[O2B-1635] Improve start sequence of server to help with big migrations and data integrity

lib/application.js

@@ -65,6 +65,20 @@ class BookkeepingApplication {
         }
     }
 
+    /**
+     * Connects to the database and applies pending migrations, then, and only then, starts the application.
+     * Servers, consumers and synchronizers must not access the database while migrations are being applied
+     *
+     * @returns {Promise<boolean>} resolves with true if the application has been started, false if the database could not be prepared
+     */
+    async start() {
+        if (!await this.connectDatabase()) {
+            return false;
+        }
+        await this.run();
+        return true;
+    }
+
     /**
      * Causes the application to be scheduled for execution.
      *

lib/main.js

@@ -13,8 +13,12 @@
 
 const application = require('./application');
 
-application.run();
-application.connectDatabase();
+application.start().then((started) => {
+    if (!started) {
+        // Nothing has been started yet (database unavailable or migration failed), there is nothing to stop gracefully
+        process.exit(1);
+    }
+});
 
 /*
  * 'SIGTERM' and 'SIGINT' have default handlers on non-Windows platforms that reset the terminal mode before exiting

test/index.js

@@ -11,6 +11,7 @@
  * or submit itself to any jurisdiction.
  */
 
+const ApplicationSuite = require('./lib/application.test.js');
 const DatabaseSuite = require('./lib/database');
 const PresentationSuite = require('./presentation');
 const PublicSuite = require('./lib/public');
@@ -22,19 +23,13 @@ const GrpcSuite = require('./gRPC');
 const APISuite = require('./api');
 const FrontendSuite = require('./public');
 
-const application = require('../lib/application');
+const { useApplication } = require('./utilities/useApplication.js');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Unit Suite', () => {
+        describe('Application', ApplicationSuite);
         describe('Database', DatabaseSuite);
         describe('Presentation', PresentationSuite);
         describe('Public', PublicSuite);

test/lib/application.test.js

@@ -0,0 +1,57 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const sinon = require('sinon');
+const { expect } = require('chai');
+const application = require('../../lib/application');
+
+module.exports = () => {
+    describe('Application start-up', () => {
+        let connectDatabaseStub;
+        let runStub;
+        let resolveMigration;
+
+        beforeEach(() => {
+            runStub = sinon.stub(application, 'run').resolves();
+            connectDatabaseStub = sinon.stub(application, 'connectDatabase');
+        });
+
+        afterEach(() => {
+            sinon.restore();
+        });
+
+        it('should not start anything (servers, synchronizers...) while migrations are still running', async () => {
+            connectDatabaseStub.returns(new Promise((resolve) => {
+                resolveMigration = resolve;
+            }));
+
+            const startPromise = application.start();
+
+            // Let any (wrongly) scheduled work run
+            await new Promise((resolve) => setImmediate(resolve));
+            expect(runStub.called).to.be.false;
+
+            resolveMigration(true);
+            expect(await startPromise).to.be.true;
+            expect(runStub.calledOnce).to.be.true;
+            expect(runStub.calledAfter(connectDatabaseStub)).to.be.true;
+        });
+
+        it('should never start the application if the database could not be prepared', async () => {
+            connectDatabaseStub.resolves(false);
+
+            expect(await application.start()).to.be.false;
+            expect(runStub.called).to.be.false;
+        });
+    });
+};

test/scripts/test-about.js

@@ -11,18 +11,11 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/about/overview.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('About', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-api.js

@@ -13,17 +13,10 @@
 
 const APISuite = require('../api');
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('API', APISuite);
 });

test/scripts/test-data-passes.js

@@ -11,19 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewPerLhcPeriodSuite = require('../public/dataPasses/overviewPerLhcPeriod.test');
 const OverviewPerSimulationPassSuite = require('../public/dataPasses/overviewPerSimulationPass.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('DataPasses', () => {
         describe('Overview Per LHC Period Page', OverviewPerLhcPeriodSuite);

test/scripts/test-envs.js

@@ -11,19 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/envs/overview.test');
 const DetailsPageSuite = require('../public/envs/detailsPage.test.js');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Envs', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-eos-report.js

@@ -11,22 +11,15 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const DcsCreationSuite = require('../public/eosReport/dcs-creation.test.js');
 const EcsCreationSuite = require('../public/eosReport/ecs-creation.test.js');
 const QcPdpCreationSuite = require('../public/eosReport/qc-pdp-creation.test.js');
 const ShiftLeaderCreationSuite = require('../public/eosReport/shift-leader-creation.test.js');
 const SlimosCreationSuite = require('../public/eosReport/slimos-creation.test.js');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('EosReport', () => {
         describe('DCS Creation Page', DcsCreationSuite);

test/scripts/test-flps.js

@@ -11,20 +11,13 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/flps/overview.test');
 // eslint-disable-next-line capitalized-comments
 const DetailSuite = require('../public/flps/detail.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Flps', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-header.js

@@ -11,18 +11,11 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const HeaderSuite = require('../public/header');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Header', HeaderSuite);
 });

test/scripts/test-home.js

@@ -11,18 +11,11 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/home/overview.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Home', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-lhc-fills.js

@@ -11,19 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/lhcFills/overview.test');
 const DetailsSuite = require('../public/lhcFills/detail.test.js');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('LhcFills', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-lhc-periods.js

@@ -11,18 +11,11 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/lhcPeriods/overview.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('lhcPeriods', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-logs.js

@@ -11,20 +11,13 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const OverviewSuite = require('../public/logs/overview.test');
 const CreateSuite = require('../public/logs/create.test');
 const DetailSuite = require('../public/logs/detail.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('Logs', () => {
         describe('Overview Page', OverviewSuite);

test/scripts/test-qc-flag-types.js

@@ -11,19 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-const application = require('../../lib/application');
+const { useApplication } = require('../utilities/useApplication.js');
 const QcFlagTypesOverviewSuite = require('../public/qcFlagTypes/overview.test');
 const QcFlagTypeCreationSuite = require('../public/qcFlagTypes/creation.test');
 
 describe('Bookkeeping', () => {
-    before(async () => {
-        await application.run();
-        await application.connectDatabase();
-    });
-
-    after(async () => {
-        await application.stop(true);
-    });
+    useApplication();
 
     describe('QcFlagTypes', () => {
         describe('Overview Page', QcFlagTypesOverviewSuite);