Rework backup & restore

.github/workflows/standard-e2e.yml

@@ -49,7 +49,13 @@ jobs:
     - name: Backup/restore tests
       timeout-minutes: 10
       run: ./test/e2e/standard/backup-restore.sh
-
+    - name: Backup test - test detection of aborted backup process - sabotage DB
+      run: psql -c 'ALTER ROLE jubilant NOSUPERUSER; REVOKE SELECT ON TABLE knex_migrations FROM jubilant;' postgresql://postgres:odktest@localhost/jubilant
+    - name: Backup test - test detection of aborted backup process
+      timeout-minutes: 2
+      run: ./test/e2e/standard/backup-abortstream.sh
+    - name: Backup test - test detection of aborted backup process - unsabotage DB
+      run: psql -c 'GRANT SELECT ON TABLE knex_migrations TO jubilant' postgresql://postgres:odktest@localhost/jubilant
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./server.log ]] || cat ./server.log"

lib/bin/restore.js

@@ -10,32 +10,109 @@
 // This script, given a path to a backup archive returned by /v1/backup, will
 // attempt to wipe the configured database and restore it from the archive.
 
-const { run } = require('../task/task');
-const { decryptFromArchive } = require('../task/fs');
+const { decryptFromLegacyArchive, getRestoreStreamFromDumpDir } = require('../util/backup-legacy');
 const tmp = require('tmp-promise');
-const { pgrestore } = require('../task/db');
 const { setlibpqEnv } = require('../util/load-db-env');
+const { checkDecrypt, getDecryptedPgRestoreStream, restoreBackupFromRestoreStream } = require('../util/backup');
+const { stdin } = require('process');
+const { createReadStream } = require('node:fs');
+const { exit } = require('node:process');
+const peek = require('buffer-peek-stream').promise;
+const { awaitSpawnee } = require('../util/process');
 
-const usage = `Usage:
-node restore.js PATH_TO_ARCHIVE PASSPHRASE
-If a passphrase was not given when the backup was created, do not give one now.`;
 
-run(async () => {
+const LEGACY_BACKUP_FORMAT_ZIP_HEADER = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+const usage = `
+Usage:
+${process.argv[0]} ${process.argv[1]} PATH_TO_ARCHIVE PASSPHRASE
+* PATH_TO_ARCHIVE may be a hyphen ("-") to indicate that the archive is being piped in from standard input.
+* If no passphrase was supplied when the backup was created, do not supply one now.
+`;
+
+
+const exitWithBadNews = (obj) => {
+  console.error(obj);
+  console.error('\n\n\n\n\t\tDATABASE RESTORE FAILED.\n\n\n\n');
+  exit(obj.exitcode || 1);
+};
+
+
+const exitOnStreamError = (stream) =>
+  stream.on('error', (streamErr) =>
+    exitWithBadNews(`stream error (${streamErr.code})`)
+  );
+
+
+const exitOnProcessStreamError = (streamingProcess) =>
+  streamingProcess.stdout.on('error', (streamErr) =>
+    exitWithBadNews(`process exited due to stream error (${streamErr.code}): ${streamingProcess.spawnargs.join(' ')}`)
+  );
+
+
+const getFormatAndStream = async (archivePath) => {
+  const instream = (archivePath === '-') ? stdin : createReadStream(archivePath);
+  const [peekbuf, reconstitutedInput] = await peek(instream, 4);
+  return [
+    peekbuf.compare(LEGACY_BACKUP_FORMAT_ZIP_HEADER) === 0,
+    reconstitutedInput,
+  ];
+};
+
+
+const restoreFromEncryptedPgDumpStream = async (instream, passphrase) => {
+  const inputCheckedForDecrypt = await checkDecrypt(instream, passphrase);
+  exitOnStreamError(inputCheckedForDecrypt);
+  const restoreStreamProc = await getDecryptedPgRestoreStream(inputCheckedForDecrypt, passphrase);
+  exitOnProcessStreamError(restoreStreamProc);
+  const restoreProc = restoreBackupFromRestoreStream(restoreStreamProc.stdout);
+  await Promise.all([awaitSpawnee(restoreStreamProc), awaitSpawnee(restoreProc)]);
+};
+
+
+const restoreFromLegacyZipFile = async (zipFilePath, passphrase) => {
+  await tmp.withDir(async (tmpdir) => {
+    await decryptFromLegacyArchive(zipFilePath, tmpdir.path, passphrase);
+    const restoreStreamProc = getRestoreStreamFromDumpDir(tmpdir.path);
+    exitOnProcessStreamError(restoreStreamProc);
+    const restoreProc = restoreBackupFromRestoreStream(restoreStreamProc.stdout);
+    await Promise.all([awaitSpawnee(restoreStreamProc), awaitSpawnee(restoreProc)]);
+  }, { unsafeCleanup: true });
+};
+
+
+const main = async () => {
   if (process.argv[2] == null) throw new Error(usage);
   const [ , , archivePath, passphrase ] = process.argv;
+  const [isLegacy, instream] = await getFormatAndStream(archivePath);
+  if (isLegacy && archivePath === '-') {
+    // The .zip needs to be seekable since the key material needed to decrypt the members are found in the last member.
+    // Technically we could spool stdin to a tempfile first, but then we'd require twice the temp space — once for the .zip,
+    // once for the pgdump dir — and it's already non-ideal to require potentially a lot of temp space
+    // for the pgdump dir state itself. Streaming-restore was never possible the legacy backup format, and going forward
+    // only new-format (streamable) backups will be made anyway, hence this decision to forgo creating a streamable
+    // path for the legacy .zip format.
+    throw new Error('Reading the backup from standard input ("-") is not supported for the legacy (.zip) backup format. Transfer the file and supply the file path instead.');
+  }
 
   setlibpqEnv(require('config').get('default.database'));
-  await tmp.withDir(async (tmpdir) => {
-    await decryptFromArchive(archivePath, tmpdir.path, passphrase);
-    await pgrestore(tmpdir.path);
-  }, { unsafeCleanup: true });
 
-  process.stdout.write(`Success. You will have to log out of the site and log back in.
-    IMPORTANT: EVERYTHING has been restored to the way things were at the time of backup, including:
+  if (isLegacy) {
+    await restoreFromLegacyZipFile(archivePath, passphrase);
+  } else {
+    await restoreFromEncryptedPgDumpStream(instream, passphrase);
+  }
+
+  return `
+
+    Success. You will have to log out of the site, and then log back in.
+    IMPORTANT: Everything has been restored to the way things were at the time of backup, including:
     * all passwords and email addresses.
-    * anything deleted since the backup was made now exists again.
-    * your backup settings.
-    Please revisit all of these and make sure they are okay.\n`);
-  return { success: true };
-});
+    * anything deleted since the backup was made — such things now exist again.
+    Please revisit all of these and make sure they are okay.
+
+  `;
+};
 
+main().catch(err => {
+  exitWithBadNews(err);
+}).then(console.log);

lib/resources/backup.js

@@ -7,155 +7,20 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { createReadStream, readdir, unlink } = require('fs');
-const { Transform } = require('stream');
-const { promisify } = require('util');
-const { randomFillSync } = require('crypto');
-const { basename, join } = require('path');
-const { mergeRight } = require('ramda');
-const archiver = require('archiver');
-const tmp = require('tmp-promise');
 const { Config } = require('../model/frames');
-const { pgdump } = require('../task/db');
-const { generateManagedKey, generateLocalCipherer } = require('../util/crypto');
-const { block } = require('../util/promise');
-const { PartialPipe } = require('../util/stream');
-const { noop } = require('../util/util');
+const { getEncryptedPgDumpStream } = require('../util/backup');
 
 
-// we have one strange problem to solve in order to achieve ad-hoc backups:
-// pgdump can take several minutes, and a tcp connection can time out after
-// as soon as 30 seconds.
-//
-// the first approach here relied on streaming keepalive http headers, which
-// worked in general but not with nginx, which wants to for whatever reason
-// buffer all response headers until the actual response begins. can't change
-// it. and doing it that way we had to call private node methods.
-//
-// this is a different approach. we first (down below in backup()) shove 32KB
-// of random data at the very start of the zip. we know this will flush through
-// the archive transform stream. now we just want to slow down that junk data
-// to buy us time for the real data to be ready.
-//
-// this trickler stream traps all data passing through it, trickling it out 128
-// bytes at a time every 5 seconds. once trickling is no longer required, it
-// forwards all data out as quickly as possible. the `released` flag is set to
-// true and the interval is cancelled when trickling is let go.
-//
-// partly as an optimization and partly to ensure our keepalive bursts are all
-// > 100 bytes, we immediately flush out all front matter until we get the actual
-// random data, which comes in as a 16KB highwatermark chunk followed by a second
-// slightly smaller chunk. this is managed by the `trapped` flag.
-const trickler = () => {
-  const chunks = [];
-
-  let trapped = false; // have we seen the first large chunk yet?
-  let released = false; // do we no longer need to trickle?
-  const transform = new Transform({
-    transform(data, _, done) {
-      // shortcut out if we don't need to trickle, or if we have some tiny
-      // header material. otherwise, put the data in our own buffer.
-      if (released) return done(null, data);
-      if (!trapped && (data.length < 128)) return done(null, data);
-
-      trapped = true;
-      chunks.push(data);
-      done();
-    }
-  });
-
-  // every 5 seconds, flush up to 128 bytes of data.
-  let ptr = 0;
-  const timer = setInterval(() => {
-    if (chunks.length === 0) return; // really shouldn't be possible but hey
-    if (transform.destroyed || transform.writableEnded)
-      return clearInterval(timer); // we don't have anything to write to anymore. bail.
-
-    const out = chunks[0];
-    if ((ptr + 128) >= out.length) {
-      transform.push(out.slice(ptr));
-      chunks.shift();
-      ptr = 0;
-    } else {
-      transform.push(out.slice(ptr, ptr + 128));
-      ptr += 128;
-    }
-  }, 5000);
-
-  // indicate that we are out of trickling and flush all buffered data.
-  const release = () => {
-    released = true;
-    clearInterval(timer);
-
-    if (chunks.length === 0) return; // whew ?
-    transform.push(chunks[0].slice(ptr));
-    for (let idx = 1; idx < chunks.length; idx += 1) transform.push(chunks[idx]);
-  };
-
-  return [ transform, release ];
-};
-
-
-const backup = (keys, response) => {
-  // set headers
-  response.set('Content-Disposition', `attachment; filename="central-backup-${(new Date()).toISOString()}.zip"`);
-  response.set('Content-Type', 'application/zip');
-
-  // set up our response stream pipeline. we'll need the trickler to keep the
-  // connection alive while we process the pgdump.
-  const zipStream = archiver('zip', { zlib: { level: -1 } });
-  const [ trickle, release ] = trickler();
-
-  // add a 32KB file full of random data to the very start of the archive. we
-  // need the randomness to be sure zip can't compress it smaller than a node
-  // buffer block (16KB).
-  const keepalive = Buffer.allocUnsafe(32000);
-  randomFillSync(keepalive);
-  zipStream.append(keepalive, { name: 'keepalive' });
-
-  // get our configuration and obtain a tmpdir to dump the database into.
-  // we don't bother catching any errors here since we can't return any kind of
-  // useful message to the user anyway, and there is nothing to clean up.
-  tmp.withDir((tmpdir) => pgdump(tmpdir.path).then(() => {
-    // halt the keepalive.
-    release();
-
-    // TODO: mostly copypasta for now from lib/task/fs:
-    return promisify(readdir)(tmpdir.path).then((files) => {
-      // create a cipher-generator for use below.
-      const [ localkey, cipherer ] = generateLocalCipherer(keys);
-      const local = { key: localkey, ivs: {} };
-
-      // add each file generated by the backup process.
-      for (const file of files) {
-        const filePath = join(tmpdir.path, file);
-        const [ iv, cipher ] = cipherer();
-        local.ivs[basename(file)] = iv.toString('base64');
-
-        const readStream = createReadStream(filePath);
-        zipStream.append(readStream.pipe(cipher), { name: file });
-        readStream.on('end', () => { unlink(filePath, noop); }); // free things up as we can
-      }
-      zipStream.append(JSON.stringify(mergeRight(keys, { local })), { name: 'keys.json' });
-
-      // this promise return lock management controls the cleanup of the zipfile,
-      // not the return of the result. that is set up as soon as .finalize() is called.
-      const [ lock, unlock ] = block();
-      zipStream.on('finish', unlock);
-      zipStream.on('error', unlock);
-      zipStream.finalize();
-      return lock;
-    });
-  }).catch(() => { response.destroy(); })); // terminate if the pgdump fails.
-
-  return PartialPipe.of(zipStream, trickle);
+const backup = (passphrase, response) => {
+  response.set('Content-Disposition', `attachment; filename="central-backup-${(new Date()).toISOString()}.pgdump.enc.bin"`);
+  response.set('Content-Type', 'application/octet-stream');
+  return getEncryptedPgDumpStream(passphrase);
 };
 
 
 module.exports = (service, endpoint) => {
   service.post('/backup', endpoint((_, { auth, body }, __, response) =>
     auth.canOrReject('backup.run', Config.species)
-      .then(() => generateManagedKey(body.passphrase))
-      .then((keys) => backup(keys, response))));
+      .then(() => backup(body.passphrase, response))));
 };