Extend helm to support Serverless functions and Analytics (cvat-ai#110)

.github/workflows/full.yml

@@ -185,7 +185,7 @@ jobs:
           LOGS_DIR: "${{ github.workspace }}/rest_api"
         run: |
             mkdir $LOGS_DIR
-            docker logs test_cvat_1 > $LOGS_DIR/cvat.log
+            docker logs test_cvat_server_1 > $LOGS_DIR/cvat.log
             docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
 
       - name: Uploading "cvat" container logs as an artifact

.github/workflows/main.yml

@@ -151,7 +151,7 @@ jobs:
           LOGS_DIR: "${{ github.workspace }}/rest_api"
         run: |
             mkdir $LOGS_DIR
-            docker logs test_cvat_1 > $LOGS_DIR/cvat.log
+            docker logs test_cvat_server_1 > $LOGS_DIR/cvat.log
             docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
 
       - name: Uploading "cvat" container logs as an artifact
@@ -288,7 +288,7 @@ jobs:
           DJANGO_SU_EMAIL: '[email protected]'
           DJANGO_SU_PASSWORD: '12qwaszx'
         run: |
-          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+          docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
 
           cd ./tests
           yarn --frozen-lockfile

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - OpenCV.js caching and autoload (<https://github.com/cvat-ai/cvat/pull/30>)
 - Publishing dev version of CVAT docker images (<https://github.com/cvat-ai/cvat/pull/53>)
 - Support of Human Pose Estimation, Facial Landmarks (and similar) use-cases, new shape type: Skeleton (<https://github.com/cvat-ai/cvat/pull/1>)
+- Added helm chart support for serverless functions and analytics (<https://github.com/cvat-ai/cvat/pull/110>)
 
 ### Changed
 - Bumped nuclio version to 1.8.14

Dockerfile

@@ -139,14 +139,16 @@ COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.g
 # Copy python virtual environment and FFmpeg binaries from build-image
 COPY --from=build-image /opt/venv /opt/venv
 ENV PATH="/opt/venv/bin:${PATH}"
+ENV NUMPROCS=1
 COPY --from=build-image /opt/ffmpeg /usr
 
 # Install and initialize CVAT, copy all necessary files
 COPY --chown=${USER} components /tmp/components
+COPY --chown=${USER} supervisord/ ${HOME}/supervisord
 COPY --chown=${USER} ssh ${HOME}/.ssh
-COPY --chown=${USER} supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
-COPY --chown=${USER} cvat/ ${HOME}/cvat
+COPY --chown=${USER} mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
 COPY --chown=${USER} utils/ ${HOME}/utils
+COPY --chown=${USER} cvat/ ${HOME}/cvat
 
 # RUN all commands below as 'django' user
 USER ${USER}
@@ -156,3 +158,4 @@ RUN mkdir data share media keys logs /tmp/supervisord
 
 EXPOSE 8080
 ENTRYPOINT ["/usr/bin/supervisord"]
+CMD ["-c", "supervisord/all.conf"]

components/analytics/docker-compose.analytics.yml

@@ -23,13 +23,15 @@ services:
       args:
         ELK_VERSION: 6.8.23
     depends_on: ['elasticsearch']
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
     restart: always
 
   cvat_kibana_setup:
     container_name: cvat_kibana_setup
     image: cvat/server:${CVAT_VERSION:-latest}
     volumes: ['./components/analytics/kibana:/home/django/kibana:ro']
-    depends_on: ['cvat']
+    depends_on: ['cvat_server']
     working_dir: '/home/django'
     networks:
       - cvat
@@ -72,7 +74,7 @@ services:
     depends_on: ['elasticsearch']
     restart: always
 
-  cvat:
+  cvat_server:
     environment:
       DJANGO_LOG_SERVER_HOST: logstash
       DJANGO_LOG_SERVER_PORT: 8080

components/analytics/kibana/kibana.yml

@@ -1,5 +1,4 @@
 server.host: 0.0.0.0
-elasticsearch.url: http://elasticsearch:9200
 elasticsearch.requestHeadersWhitelist: ['cookie', 'authorization', 'x-forwarded-user']
 kibana.defaultAppId: 'discover'
 server.basePath: /analytics

components/analytics/kibana_conf.yml

@@ -21,7 +21,7 @@ http:
   middlewares:
     analytics-auth:
       forwardauth:
-        address: http://cvat:8080/analytics
+        address: http://cvat_server:8080/analytics
         authRequestHeaders:
           - "Cookie"
           - "Authorization"

components/analytics/logstash/logstash.yml

@@ -1,3 +1,4 @@
 queue.type: persisted
 queue.max_bytes: 1gb
 queue.checkpoint.writes: 20
+http.host: 0.0.0.0

components/serverless/docker-compose.serverless.yml

@@ -18,7 +18,7 @@ services:
     ports:
       - '8070:8070'
 
-  cvat:
+  cvat_server:
     environment:
       CVAT_SERVERLESS: 1
 

cvat/apps/engine/backup.py

@@ -839,7 +839,7 @@ def _import(importer, request, rq_id, Serializer, file_field_name, location_conf
                 serializer = Serializer(data=request.data)
                 serializer.is_valid(raise_exception=True)
                 payload_file = serializer.validated_data[file_field_name]
-                fd, filename = mkstemp(prefix='cvat_')
+                fd, filename = mkstemp(prefix='cvat_', dir=settings.TMP_FILES_ROOT)
                 with open(filename, 'wb+') as f:
                     for chunk in payload_file.chunks():
                         f.write(chunk)
@@ -863,9 +863,10 @@ def _import_from_cloud_storage(storage, file_name):
 
             data = _import_from_cloud_storage(storage, file_name)
 
-            fd, filename = mkstemp(prefix='cvat_')
+            fd, filename = mkstemp(prefix='cvat_', dir=settings.TMP_FILES_ROOT)
             with open(filename, 'wb+') as f:
                 f.write(data.getbuffer())
+
         rq_job = queue.enqueue_call(
             func=importer,
             args=(filename, request.user.id, org_id),

cvat/apps/engine/views.py

@@ -2096,7 +2096,7 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name,
                 serializer = AnnotationFileSerializer(data=request.data)
                 if serializer.is_valid(raise_exception=True):
                     anno_file = serializer.validated_data['annotation_file']
-                    fd, filename = mkstemp(prefix='cvat_{}'.format(pk))
+                    fd, filename = mkstemp(prefix='cvat_{}'.format(pk), dir=settings.TMP_FILES_ROOT)
                     with open(filename, 'wb+') as f:
                         for chunk in anno_file.chunks():
                             f.write(chunk)
@@ -2114,7 +2114,7 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name,
 
                 data = _import_from_cloud_storage(storage, filename)
 
-                fd, filename = mkstemp(prefix='cvat_')
+                fd, filename = mkstemp(prefix='cvat_{}'.format(pk), dir=settings.TMP_FILES_ROOT)
                 with open(filename, 'wb+') as f:
                     f.write(data.getbuffer())
 
@@ -2262,7 +2262,7 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name, filename=N
             serializer = DatasetFileSerializer(data=request.data)
             if serializer.is_valid(raise_exception=True):
                 dataset_file = serializer.validated_data['dataset_file']
-                fd, filename = mkstemp(prefix='cvat_{}'.format(pk))
+                fd, filename = mkstemp(prefix='cvat_{}'.format(pk), dir=settings.TMP_FILES_ROOT)
                 with open(filename, 'wb+') as f:
                     for chunk in dataset_file.chunks():
                         f.write(chunk)
@@ -2281,7 +2281,7 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name, filename=N
 
             data = _import_from_cloud_storage(storage, filename)
 
-            fd, filename = mkstemp(prefix='cvat_')
+            fd, filename = mkstemp(prefix='cvat_', dir=settings.TMP_FILES_ROOT)
             with open(filename, 'wb+') as f:
                 f.write(data.getbuffer())
 

cvat/apps/lambda_manager/views.py

@@ -41,14 +41,16 @@ class LambdaGateway:
     NUCLIO_ROOT_URL = '/api/functions'
 
     def _http(self, method="get", scheme=None, host=None, port=None,
-        url=None, headers=None, data=None):
+        function_namespace=None, url=None, headers=None, data=None):
         NUCLIO_GATEWAY = '{}://{}:{}'.format(
             scheme or settings.NUCLIO['SCHEME'],
             host or settings.NUCLIO['HOST'],
             port or settings.NUCLIO['PORT'])
+        NUCLIO_FUNCTION_NAMESPACE = function_namespace or settings.NUCLIO['FUNCTION_NAMESPACE']
         extra_headers = {
             'x-nuclio-project-name': 'cvat',
-            'x-nuclio-function-namespace': 'nuclio',
+            'x-nuclio-function-namespace': NUCLIO_FUNCTION_NAMESPACE,
+            'x-nuclio-invoke-via': 'domain-name',
         }
         if headers:
             extra_headers.update(headers)
@@ -77,6 +79,13 @@ def get(self, func_id):
         return response
 
     def invoke(self, func, payload):
+        if os.getenv('KUBERNETES_SERVICE_HOST'):
+            return self._http(method="post", url='/api/function_invocations',
+            data=payload, headers={
+                'x-nuclio-function-name': func.id,
+                'x-nuclio-path': '/'
+            })
+
         # Note: call the function directly without the nuclio dashboard
         # host.docker.internal for Linux will work only with Docker 20.10+
         NUCLIO_TIMEOUT = settings.NUCLIO['DEFAULT_TIMEOUT']

cvat/settings/base.py

@@ -284,7 +284,8 @@ def add_ssh_keys():
     'SCHEME': os.getenv('CVAT_NUCLIO_SCHEME', 'http'),
     'HOST': os.getenv('CVAT_NUCLIO_HOST', 'localhost'),
     'PORT': os.getenv('CVAT_NUCLIO_PORT', 8070),
-    'DEFAULT_TIMEOUT': os.getenv('CVAT_NUCLIO_DEFAULT_TIMEOUT', 120)
+    'DEFAULT_TIMEOUT': os.getenv('CVAT_NUCLIO_DEFAULT_TIMEOUT', 120),
+    'FUNCTION_NAMESPACE': os.getenv('CVAT_NUCLIO_FUNCTION_NAMESPACE', 'nuclio'),
 }
 
 RQ_SHOW_ADMIN_LINK = True

docker-compose.ci.yml

@@ -13,7 +13,7 @@ services:
       context: .
       dockerfile: Dockerfile.ci
     depends_on:
-      - cvat
+      - cvat_server
     environment:
       COVERALLS_SERVICE_NAME:
       CONTAINER_COVERAGE_DATA_DIR:

docker-compose.dev.yml

@@ -6,7 +6,7 @@
 version: '3.3'
 
 services:
-  cvat:
+  cvat_server:
     build:
       context: .
       args:

docker-compose.yml

@@ -25,8 +25,8 @@ services:
     networks:
       - cvat
 
-  cvat:
-    container_name: cvat
+  cvat_server:
+    container_name: cvat_server
     image: cvat/server:${CVAT_VERSION:-latest}
     restart: always
     depends_on:
@@ -40,6 +40,8 @@ services:
       CVAT_POSTGRES_HOST: 'cvat_db'
       ADAPTIVE_AUTO_ANNOTATION: 'false'
       no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy}
+      NUMPROCS: 1
+    command: -c supervisord/server.conf
     labels:
       - traefik.enable=true
       - traefik.http.services.cvat.loadbalancer.server.port=8080
@@ -55,12 +57,74 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  cvat_utils:
+    container_name: cvat_utils
+    image: cvat/server:${CVAT_VERSION:-latest}
+    restart: always
+    depends_on:
+      - cvat_redis
+      - cvat_db
+      - cvat_opa
+    environment:
+      CVAT_REDIS_HOST: 'cvat_redis'
+      CVAT_POSTGRES_HOST: 'cvat_db'
+      no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy}
+    command: -c supervisord/utils.conf
+    volumes:
+      - cvat_data:/home/django/data
+      - cvat_keys:/home/django/keys
+      - cvat_logs:/home/django/logs
+    networks:
+      - cvat
+
+  cvat_worker_default:
+    container_name: cvat_worker_default
+    image: cvat/server:${CVAT_VERSION:-latest}
+    restart: always
+    depends_on:
+      - cvat_redis
+      - cvat_db
+      - cvat_opa
+    environment:
+      CVAT_REDIS_HOST: 'cvat_redis'
+      CVAT_POSTGRES_HOST: 'cvat_db'
+      no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy}
+      NUMPROCS: 2
+    command: -c supervisord/worker.default.conf
+    volumes:
+      - cvat_data:/home/django/data
+      - cvat_keys:/home/django/keys
+      - cvat_logs:/home/django/logs
+    networks:
+      - cvat
+
+  cvat_worker_low:
+    container_name: cvat_worker_low
+    image: cvat/server:${CVAT_VERSION:-latest}
+    restart: always
+    depends_on:
+      - cvat_redis
+      - cvat_db
+      - cvat_opa
+    environment:
+      CVAT_REDIS_HOST: 'cvat_redis'
+      CVAT_POSTGRES_HOST: 'cvat_db'
+      no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy}
+      NUMPROCS: 1
+    command: -c supervisord/worker.low.conf
+    volumes:
+      - cvat_data:/home/django/data
+      - cvat_keys:/home/django/keys
+      - cvat_logs:/home/django/logs
+    networks:
+      - cvat
+
   cvat_ui:
     container_name: cvat_ui
     image: cvat/ui:${CVAT_VERSION:-latest}
     restart: always
     depends_on:
-      - cvat
+      - cvat_server
     labels:
       - traefik.enable=true
       - traefik.http.services.cvat-ui.loadbalancer.server.port=80

helm-chart/Chart.yaml

@@ -26,10 +26,36 @@ appVersion: latest
 
 dependencies:
   - name: redis
-    version: "12.9.*"
+    version: "16.13.*"
     repository: https://charts.bitnami.com/bitnami
     condition: redis.enabled
+
   - name: postgresql
-    version: "10.3.*"
+    version: "10.16.*"
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
+
+  - name: nuclio
+    version: 0.12.1
+    repository: https://nuclio.github.io/nuclio/charts
+    condition: nuclio.enabled
+
+  - name: logstash
+    version: "6.8.*"
+    repository: https://helm.elastic.co
+    condition: analytics.enabled
+
+  - name: elasticsearch
+    version: "6.8.*"
+    repository: https://helm.elastic.co
+    condition: analytics.enabled
+
+  - name: kibana
+    version: "6.8.*"
+    repository: https://helm.elastic.co
+    condition: analytics.enabled
+
+  - name: traefik
+    version: 10.24.0
+    repository: https://helm.traefik.io/traefik
+    condition: ingress.enabled