refactor to serve the compilation artifacts from the docker image (#3401)

* refactor to serve the compilation artifacts from the docker image

* update CI workflow

* check in an empty artifacts/ dir

* only serve from artifacts/ if the directory exists

* remove extra build-storage-artifacts step

* remove gcs upload instructions

* add caching for 1 hour

Author: devoncarew

.cloud_build/README.md

@@ -35,7 +35,6 @@ gcloud builds submit \
 
 The substitutions are a comma separated list of `SUBSITUTION=value`:
 - `_FLUTTER_CHANNEL` - The Flutter channel to use. (`stable`, `beta`, or `main`)
-- `_STORAGE_BUCKET` - The name of the Cloud Storage bucket to use (nnbd_artifacts)
 - `_SERVICE_NAME` - The name of the Cloud Run service (dart-services-cloud-run, flutter-beta-channel, or flutter-master-channel)
 - `_DEPLOY_REGION` - The region to deploy the Cloud Run service to (us-central1)
 - `_REDIS_ADDR` - the IP address of the Redis bucket (10.0.0.4:6379)

.cloud_build/dart_services.yaml

@@ -28,17 +28,6 @@ steps:
       - FLUTTER_CHANNEL=${_FLUTTER_CHANNEL}
     dir: pkgs/dart_services
 
-  # Build storage artifacts
-  - name: gcr.io/$PROJECT_ID/flutter:${_FLUTTER_CHANNEL}
-    id: Build storage artifacts
-    entrypoint: dart
-    args:
-      - tool/grind.dart
-      - build-storage-artifacts
-    env:
-      - FLUTTER_CHANNEL=${_FLUTTER_CHANNEL}
-    dir: pkgs/dart_services
-
   # Save flutter version to a file
   - name: gcr.io/$PROJECT_ID/flutter:${_FLUTTER_CHANNEL}
     id: Save flutter version
@@ -51,30 +40,6 @@ steps:
       - FLUTTER_CHANNEL=${_FLUTTER_CHANNEL}
     dir: pkgs/dart_services
 
-  # Upload storage artifacts
-  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
-    id: Upload storage artifacts
-    entrypoint: bash
-    args:
-      - -c
-      - |
-          echo "Uploading storage artifact for version $(cat /workspace/dart_version.txt)" && gsutil -h "Cache-Control: public, max-age=604800, immutable" cp -z js artifacts/*.js* gs://${_STORAGE_BUCKET}/$(cat /workspace/dart_version.txt)/
-    env:
-      - FLUTTER_CHANNEL=${_FLUTTER_CHANNEL}
-    dir: pkgs/dart_services
-
-  # Validate storage artifacts
-  - name: gcr.io/$PROJECT_ID/flutter:${_FLUTTER_CHANNEL}
-    id: Validate storage artifacts
-    entrypoint: dart
-    args:
-      - tool/grind.dart
-      - validate-storage-artifacts
-      - --bucket=${_STORAGE_BUCKET}
-    env:
-      - FLUTTER_CHANNEL=${_FLUTTER_CHANNEL}
-    dir: pkgs/dart_services
-
   # Build the container image
   - name: 'gcr.io/cloud-builders/docker'
     args:
@@ -118,7 +83,7 @@ steps:
       - --cpu-boost
       - --max-instances=1000
       - --command=bin/server
-      - --args=--redis-url=redis://${_REDIS_ADDR},--storage-bucket=${_STORAGE_BUCKET}
+      - --args=--redis-url=redis://${_REDIS_ADDR}
     dir: pkgs/dart_services
 timeout: 1200s
 images:

.github/workflows/dart_services.yml

@@ -42,13 +42,13 @@ jobs:
         run: dart format --set-exit-if-changed bin lib test tool
 
       - name: Create template projects and populate the custom pub cache
-        run: dart run grinder build-project-templates
+        run: dart tool/grind.dart build-project-templates
 
       # Build the web artifacts on the bots to ensure they don't regress.
       - name: Build web artifacts
-        run: dart run grinder build-storage-artifacts
+        run: dart tool/grind.dart build-storage-artifacts
 
       - name: Run tests
         run: dart test test/presubmit
 
-      - run: sh ./tool/check_cycles.sh
+      - run: tool/check_cycles.sh

.github/workflows/dartpad_shared.yml

@@ -37,4 +37,4 @@ jobs:
       - name: Analyze project source
         run: dart analyze --fatal-infos
 
-      - run: sh ./tool/check_cycles.sh
+      - run: tool/check_cycles.sh

.github/workflows/dartpad_ui.yml

@@ -35,9 +35,12 @@ jobs:
       - run: flutter pub get
       - run: flutter analyze
       - run: dart format --set-exit-if-changed .
-      - name: Create template projects and populate the custom pub cache
-        run: dart run grinder build-project-templates
+
+      - name: Create template projects
+        run: dart tool/grind.dart build-project-templates
         working-directory: pkgs/dart_services/
+
       - run: flutter build web
-      - run: flutter test test
-      - run: sh ./tool/check_cycles.sh
+      - run: flutter test
+
+      - run: tool/check_cycles.sh

.github/workflows/samples.yml

@@ -30,4 +30,4 @@ jobs:
       - run: dart format --set-exit-if-changed .
       - run: dart tool/samples.dart --verify
       - run: flutter build web
-      - run: sh ./tool/check_cycles.sh
+      - run: tool/check_cycles.sh

pkgs/dart_services/.gitignore

@@ -19,7 +19,7 @@ pubspec.lock
 dump.rdb
 
 # compilation artifacts
-artifacts/
+artifacts/**
 project_templates/
 flutter-sdks/
 local_pub_cache/

pkgs/dart_services/Dockerfile

@@ -12,7 +12,8 @@ WORKDIR /pkgs/dart_services
 RUN dart pub get
 RUN dart compile exe bin/server.dart -o bin/server
 
-RUN dart run grinder build-project-templates
+RUN dart tools/grind.dart build-project-templates
+RUN dart tools/grind.dart build-storage-artifacts
 
 ENV BUILD_SHA=$BUILD_SHA
 

pkgs/dart_services/README.md

@@ -29,7 +29,6 @@ dart bin/server.dart
 The server will run from port 8080 and export several JSON APIs, like
 `/api/v3/analyze` and `/api/v3/compile`.
 
-
 ### Enabling code generation
 
 To test code generation features locally:
@@ -61,12 +60,6 @@ are uploaded to Cloud Storage automatically. These files are located in the
 
 If you need to re-generate these files, run the following command.
 
-```
-grind build-storage-artifacts
-```
-
-Or, if you don't have `grind` on your PATH:
-
 ```
 dart tool/grind.dart build-storage-artifacts
 ```
@@ -82,18 +75,13 @@ that the dependencies resolve and update the pinned versions specified in the
    whitelisted list of packages.
 2. Create the Dart and Flutter projects in the `project_templates/` directory:
 
-    ```bash
-    grind build-project-templates
-    ```
-
-    Or, if you don't have `grind` on your PATH:
 
-   ```
+   ```bash
    dart tool/grind.dart build-project-templates
    ```
 
 4. Run `pub upgrade` in the Dart or Flutter project in `project_templates/`
-5. Run `grind update-pub-dependencies` to overwrite the
+5. Run `dart tool/grind.dart update-pub-dependencies` to overwrite the
    `tool/dependencies/pub_dependencies_<CHANNEL>.yaml` file for your current
    channel. Or, if you don't have `grind` on your PATH, `dart tool/grind.dart update-pub-dependencies`
 6. Repeat the above steps for the latest version of each Flutter channel

pkgs/dart_services/lib/server.dart

@@ -27,12 +27,6 @@ Future<void> main(List<String> args) async {
   final parser = ArgParser()
     ..addOption('port', valueHelp: 'port', help: 'The port to listen on.')
     ..addOption('redis-url', valueHelp: 'url', help: 'The redis server url.')
-    ..addOption(
-      'storage-bucket',
-      valueHelp: 'name',
-      help: 'The name of the Cloud Storage bucket for compilation artifacts.',
-      defaultsTo: 'nnbd_artifacts',
-    )
     ..addFlag(
       'help',
       abbr: 'h',
@@ -68,8 +62,6 @@ Future<void> main(List<String> args) async {
   emitLogsToStdout();
 
   final redisServerUri = results['redis-url'] as String?;
-  final storageBucket =
-      results['storage-bucket'] as String? ?? 'nnbd_artifacts';
 
   final cloudRunEnvVars = Platform.environment.entries
       .where((entry) => entry.key.startsWith('K_'))
@@ -89,8 +81,7 @@ Starting dart-services:
   final server = await EndpointsServer.serve(
     port,
     sdk,
-    redisServerUri,
-    storageBucket,
+    redisServerUri: redisServerUri,
   );
 
   _logger.genericInfo('Listening on port ${server.port}');
@@ -99,15 +90,10 @@ Starting dart-services:
 class EndpointsServer {
   static Future<EndpointsServer> serve(
     int port,
-    Sdk sdk,
-    String? redisServerUri,
-    String storageBucket,
-  ) async {
-    final endpointsServer = EndpointsServer._(
-      sdk,
-      redisServerUri,
-      storageBucket,
-    );
+    Sdk sdk, {
+    required String? redisServerUri,
+  }) async {
+    final endpointsServer = EndpointsServer._(sdk, redisServerUri);
     await endpointsServer._init();
 
     endpointsServer.server = await shelf.serve(
@@ -125,7 +111,7 @@ class EndpointsServer {
 
   late final CommonServerApi commonServer;
 
-  EndpointsServer._(Sdk sdk, String? redisServerUri, String storageBucket) {
+  EndpointsServer._(Sdk sdk, String? redisServerUri) {
     // The name of the Cloud Run revision being run, for more detail please see:
     // https://cloud.google.com/run/docs/reference/container-contract#env-vars
     final serverVersion = Platform.environment['K_REVISION'];
@@ -134,9 +120,7 @@ class EndpointsServer {
         ? NoopCache()
         : RedisCache(redisServerUri, sdk, serverVersion);
 
-    commonServer = CommonServerApi(
-      CommonServerImpl(sdk, cache, storageBucket: storageBucket),
-    );
+    commonServer = CommonServerApi(CommonServerImpl(sdk, cache));
 
     final pipeline = const Pipeline()
         .addMiddleware(logRequestsToLogger(_logger))
@@ -196,7 +180,7 @@ class TestServerRunner {
 
     _started = Completer<void>();
     try {
-      await EndpointsServer.serve(_port, sdk, null, 'nnbd_artifacts');
+      await EndpointsServer.serve(_port, sdk, redisServerUri: null);
     } on SocketException {
       // This is expected if the server is already running.
     }