Add Evaluate Measure Timeout

The environment variable FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT can be
used to set a evaluate measure timeout in milliseconds. The default
value will be 1 hour in order to prevent overly long evaluations from
slowing down Blaze. At the same time the default should not change the
behaviour of normal evaluations.

.github/scripts/evaluate-measure-timeout.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+BASE="http://localhost:8080/fhir"
+NAME="$1"
+
+DIAGNOSTICS=$(blazectl --server "$BASE" evaluate-measure ".github/scripts/cql/$NAME.yml" 2> /dev/null | grep Diagnostics | cut -d: -f2 | xargs)
+
+if [ "$DIAGNOSTICS" = "Timeout of 10 millis eclipsed while evaluating." ]; then
+  echo "Success: timeout happened"
+else
+  echo "Fail: no timeout"
+  exit 1
+fi

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Setup Clojure
       uses: DeLaGuardo/setup-clojure@master
       with:
-        clj-kondo: '2022.10.05'
+        clj-kondo: '2023.01.20'
 
     - name: Check out Git repository
       uses: actions/checkout@v3
@@ -645,6 +645,44 @@ jobs:
     - name: Load Data
       run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/big-tx
 
+  evaluate-measure-timeout-test:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@v3
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@v3
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT=10 -p 8080:8080 -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Docker Logs
+      run: docker logs blaze
+
+    - name: Check Capability Statement
+      run: .github/scripts/check-capability-statement.sh
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Evaluate CQL Query 1
+      run: .github/scripts/evaluate-measure-timeout.sh q1
+
   include-without-referential-integrity-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -1203,6 +1241,7 @@ jobs:
     - not-enforcing-referential-integrity-test
     - small-transactions-test
     - big-transaction-test
+    - evaluate-measure-timeout-test
     - include-without-referential-integrity-test
     - chaining-without-referential-integrity-test
     - bundle-with-references-test

docs/cql-queries.md

@@ -11,7 +11,7 @@ If you like to use the command line, please look into [this section](cql-queries
 
 ## API Documentation
 
-If yopu like to use the CQL Evaluation API directly, please read the [CQL API Documentation](cql-queries/api.md).
+If you'd like to use the CQL Evaluation API directly, please read the [CQL API Documentation](cql-queries/api.md).
 
 ## Install the Quality Reporting UI
 

docs/deployment/environment-variables.md

@@ -86,6 +86,7 @@ More information about distributed deployment are available [here](distributed.m
 | LOG_LEVEL                               | info                       | v0.6   | —      | one of trace, debug, info, warn or error                                                       |
 | JAVA_TOOL_OPTIONS                       | —                          | —      | —      | JVM options \(Docker only\)                                                                    |
 | FHIR_OPERATION_EVALUATE_MEASURE_THREADS | 4                          | v0.8   | —      | The maximum number of parallel $evaluate-measure executions.                                   |
+| FHIR_OPERATION_EVALUATE_MEASURE_TIMEOUT | 3600000 (1h)               | v0.19  | —      | Timeout in milliseconds for $evaluate-measure executions.                                      |
 | OPENID_PROVIDER_URL                     | —                          | v0.11  | —      | [OpenID Connect][4] provider URL to enable [authentication][5]                                 |
 | ENFORCE_REFERENTIAL_INTEGRITY           | true                       | v0.14  | —      | Enforce referential integrity on resource create, update and delete.                           |
 | DB_SYNC_TIMEOUT                         | 10000                      | v0.15  | —      | Timeout in milliseconds for all reading FHIR interactions acquiring the newest database state. |

docs/performance/fhir-search/simple-code-search.sh

@@ -10,7 +10,7 @@ RESOURCE_HANDLE_CACHE_SIZE=30000000
 start-blaze() {
   echo "Starting Blaze..."
   docker run --name blaze --rm -v "$VOLUME:/app/data" \
-      -e JAVA_TOOL_OPTIONS="-Xmx${HEAP_SIZE}g -Dclojure.compiler.direct-linking=true" \
+      -e JAVA_TOOL_OPTIONS="-Xmx${HEAP_SIZE}g" \
       -e LOG_LEVEL=debug \
       -e DB_BLOCK_CACHE_SIZE=$BLOCK_CACHE_SIZE \
       -e DB_RESOURCE_CACHE_SIZE=$RESOURCE_CACHE_SIZE \
@@ -19,7 +19,7 @@ start-blaze() {
       -e DB_RESOURCE_INDEXER_THREADS=16 \
       -p 8080:8080 \
       -p 8081:8081 \
-      -d samply/blaze:pr-678
+      -d samply/blaze:0.18
 
   ../../.github/scripts/wait-for-url.sh  http://localhost:8080/health
   echo "Finished"

docs/performance/import.sh

@@ -5,14 +5,14 @@ C=8
 
 import-once() {
   docker run --name blaze --rm -v blaze-data:/app/data \
-    -e JAVA_TOOL_OPTIONS="-Xmx4g -Dclojure.compiler.direct-linking=true" \
+    -e JAVA_TOOL_OPTIONS="-Xmx4g" \
     -e LOG_LEVEL=debug \
     -e DB_BLOCK_CACHE_SIZE=8192 \
     -e DB_MAX_BACKGROUND_JOBS=16 \
     -e DB_RESOURCE_INDEXER_THREADS=16 \
     -p 8080:8080 \
     -p 8081:8081 \
-    -d samply/blaze:pr-678
+    -d samply/blaze:0.18
 
   ../../.github/scripts/wait-for-url.sh  http://localhost:8080/health
 

modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj

@@ -19,6 +19,7 @@
     [blaze.spec]
     [clojure.spec.alpha :as s]
     [integrant.core :as ig]
+    [java-time.api :as time]
     [reitit.core :as reitit]
     [ring.util.response :as ring]
     [taoensso.timbre :as log])
@@ -123,7 +124,8 @@
 
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [:blaze.db/node ::executor :blaze/clock :blaze/rng-fn]))
+  (s/keys :req-un [:blaze.db/node ::executor :blaze/clock :blaze/rng-fn]
+          :opt-un [::timeout]))
 
 
 (defmethod ig/init-key ::handler [_ context]
@@ -133,6 +135,14 @@
       (wrap-observe-request-duration "operation-evaluate-measure")))
 
 
+(defmethod ig/pre-init-spec ::timeout [_]
+  (s/keys :req-un [:blaze.fhir.operation.evaluate-measure.timeout/millis]))
+
+
+(defmethod ig/init-key ::timeout [_ {:keys [millis]}]
+  (time/millis millis))
+
+
 (defmethod ig/pre-init-spec ::executor [_]
   (s/keys :opt-un [::num-threads]))
 

modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/cql.clj

@@ -6,9 +6,11 @@
     [blaze.elm.util :as elm-util]
     [blaze.fhir.spec :as fhir-spec]
     [clojure.core.reducers :as r]
+    [cognitect.anomalies :as anom]
     [taoensso.timbre :as log])
   (:import
-    [java.lang AutoCloseable]))
+    [java.lang AutoCloseable]
+    [java.time Duration]))
 
 
 (set! *warn-on-reflection* true)
@@ -37,7 +39,7 @@
           (ex-message e)))
 
 
-(defn- evaluate-expression-1 [context subject-handle name expression]
+(defn- evaluate-expression-1* [context subject-handle name expression]
   (try
     (expr/eval context expression subject-handle)
     (catch Exception e
@@ -54,6 +56,24 @@
             (merge ex-data))))))
 
 
+(defn- timeout-millis [{:keys [timeout]}]
+  (.toMillis ^Duration timeout))
+
+
+(defn- timeout-eclipsed-msg [context]
+  (format "Timeout of %d millis eclipsed while evaluating."
+          (timeout-millis context)))
+
+
+(defn- evaluate-expression-1
+  [{:keys [timeout-eclipsed?] :as context} subject-handle name expression]
+  (if (timeout-eclipsed?)
+    {::anom/category ::anom/interrupted
+     ::anom/message (timeout-eclipsed-msg context)
+     :timeout (:timeout context)}
+    (evaluate-expression-1* context subject-handle name expression)))
+
+
 (defn- close-batch-db! [{:keys [db]}]
   (.close ^AutoCloseable db))
 
@@ -250,7 +270,7 @@
     (stratum-combine-op context)
     (fn [context {:keys [subject-handle] :as handle}]
       (if-ok [stratum (evaluate-stratum-expression context subject-handle
-                                                 name expression)]
+                                                   name expression)]
         (update context ::result stratum-result-reduce-op stratum handle)
         #(reduced (assoc context ::result %))))
     handles))

modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj

@@ -13,6 +13,7 @@
     [blaze.handler.fhir.util :as fhir-util]
     [blaze.luid :as luid]
     [clojure.string :as str]
+    [java-time.api :as time]
     [prometheus.alpha :as prom]
     [taoensso.timbre :as log])
   (:import
@@ -329,14 +330,19 @@
 
 
 (defn- enhance-context
-  [{:keys [clock db] :as context} measure {:keys [report-type subject-ref]}]
-  (let [subject-type (subject-type measure)]
+  [{:keys [clock db timeout] :as context :or {timeout (time/hours 1)}} measure
+   {:keys [report-type subject-ref]}]
+  (let [subject-type (subject-type measure)
+        now (now clock)
+        timeout-instant (time/instant (time/plus now timeout))]
     (when-ok [{:keys [expression-defs function-defs parameter-default-values]} (compile-primary-library db measure)
               subject-handle (some->> subject-ref (subject-handle db subject-type))]
       (cond->
         (assoc context
           :db db
-          :now (now clock)
+          :now now
+          :timeout-eclipsed? #(not (.isBefore (.instant ^Clock clock) timeout-instant))
+          :timeout timeout
           :expression-defs expression-defs
           :function-defs function-defs
           :parameters parameter-default-values

modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/spec.clj

@@ -1,12 +1,22 @@
 (ns blaze.fhir.operation.evaluate-measure.spec
   (:require
     [blaze.executors :as ex]
-    [clojure.spec.alpha :as s]))
+    [blaze.fhir.operation.evaluate-measure :as-alias measure]
+    [clojure.spec.alpha :as s]
+    [java-time.api :as time]))
 
 
-(s/def :blaze.fhir.operation.evaluate-measure/executor
+(s/def ::measure/executor
   ex/executor?)
 
 
-(s/def :blaze.fhir.operation.evaluate-measure/num-threads
+(s/def ::measure/num-threads
+  pos-int?)
+
+
+(s/def ::measure/timeout
+  time/duration?)
+
+
+(s/def :blaze.fhir.operation.evaluate-measure.timeout/millis
   nat-int?)

modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj

@@ -0,0 +1,31 @@
+(ns blaze.fhir.operation.evaluate-measure.cql.spec
+  (:require
+    [blaze.elm.compiler :as-alias compiler]
+    [blaze.fhir.operation.evaluate-measure.cql :as-alias cql]
+    [clojure.spec.alpha :as s]
+    [java-time.api :as time]))
+
+
+(s/def ::cql/now
+  time/offset-date-time?)
+
+
+(s/def ::cql/timeout-eclipsed?
+  ifn?)
+
+
+(s/def ::cql/timeout
+  time/duration?)
+
+
+(s/def ::cql/context
+  (s/keys :req-un [:blaze.db/db ::cql/now ::cql/timeout-eclipsed? ::cql/timeout
+                   ::compiler/expression-defs]))
+
+
+(s/def ::cql/parameters
+  (s/map-of string? any?))
+
+
+(s/def ::cql/individual-context
+  (s/merge ::cql/context (s/keys :opt-un [::cql/parameters])))

modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_spec.clj

@@ -1,67 +1,50 @@
 (ns blaze.fhir.operation.evaluate-measure.cql-spec
   (:require
     [blaze.db.spec]
-    [blaze.elm.compiler :as-alias compiler]
     [blaze.elm.compiler.library-spec]
     [blaze.elm.expression-spec]
     [blaze.fhir.operation.evaluate-measure.cql :as cql]
+    [blaze.fhir.operation.evaluate-measure.cql.spec]
     [blaze.fhir.operation.evaluate-measure.measure :as-alias measure]
     [blaze.fhir.operation.evaluate-measure.measure.spec]
     [blaze.fhir.spec]
     [clojure.spec.alpha :as s]
-    [cognitect.anomalies :as anom]
-    [java-time.api :as time]))
-
-
-(s/def ::now
-  time/offset-date-time?)
-
-
-(s/def ::parameters
-  (s/map-of string? any?))
-
-
-(s/def ::context
-  (s/keys :req-un [:blaze.db/db ::now ::compiler/expression-defs]))
-
-
-(s/def ::individual-context
-  (s/keys :req-un [:blaze.db/db ::now ::compiler/expression-defs] :opt-un [::parameters]))
+    [cognitect.anomalies :as anom]))
 
 
 (s/fdef cql/evaluate-expression
-  :args (s/cat :context ::context :name string? :subject-type :fhir.resource/type
+  :args (s/cat :context ::cql/context :name string? :subject-type :fhir.resource/type
                :population-basis (s/alt :subject-based #{:boolean} :other :fhir.resource/type))
   :ret (s/or :handles ::measure/handles
              :anomaly ::anom/anomaly))
 
 
 (s/fdef cql/evaluate-individual-expression
-  :args (s/cat :context ::individual-context
+  :args (s/cat :context ::cql/individual-context
                :subject-handle :blaze.db/resource-handle
                :name string?)
   :ret (s/or :value any?
              :anomaly ::anom/anomaly))
 
 
 (s/fdef cql/calc-strata
-  :args (s/cat :context ::context
+  :args (s/cat :context ::cql/context
                :expression-name string?
                :handles ::measure/handles)
   :ret (s/or :strata (s/map-of any? ::measure/handles)
              :anomaly ::anom/anomaly))
 
 
 (s/fdef cql/calc-function-strata
-  :args (s/cat :context ::context
+  :args (s/cat :context ::cql/context
                :function-name string?
                :handles ::measure/handles)
   :ret (s/or :strata (s/map-of any? ::measure/handles)
              :anomaly ::anom/anomaly))
 
 
 (s/fdef cql/calc-multi-component-strata
-  :args (s/cat :context ::context
+  :args (s/cat :context ::cql/context
                :expression-names (s/coll-of string?)
                :handles ::measure/handles)
   :ret (s/or :strata (s/map-of (s/coll-of any?) ::measure/handles)