Release 7.034

techascent · Nov 16, 2024 · da7fb0c · da7fb0c
1 parent 12c9a96
commit da7fb0c
Show file tree

Hide file tree

Showing 40 changed files with 62 additions and 44 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,7 @@
 # Changelog
+#7.034
+ * Reverted transit encoding of instant back to milliseconds since epoch as js api doesn't support microseconds since epoch.
+
 # 7.033
  * [issue-434](https://github.com/techascent/tech.ml.dataset/issues/413) - bad transit encoding - packed instants are microseconds since epoch and have been for a while - not milliseconds since epoch.
 

diff --git a/deps.edn b/deps.edn
@@ -1,6 +1,6 @@
 {:paths ["src" "resources" "target/classes"]
  :deps {;;org.clojure/clojure {:mvn/version "1.11.1"}
-        cnuernber/dtype-next {:mvn/version "10.120"}
+        cnuernber/dtype-next {:mvn/version "10.122"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}
@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.033"
+               :version "7.034"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

diff --git a/docs/000-getting-started.html b/docs/000-getting-started.html
diff --git a/docs/100-walkthrough.html b/docs/100-walkthrough.html
diff --git a/docs/200-quick-reference.html b/docs/200-quick-reference.html
diff --git a/docs/columns-readers-and-datatypes.html b/docs/columns-readers-and-datatypes.html
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/nippy-serialization-rocks.html b/docs/nippy-serialization-rocks.html
diff --git a/docs/supported-datatypes.html b/docs/supported-datatypes.html
diff --git a/docs/tech.v3.dataset.categorical.html b/docs/tech.v3.dataset.categorical.html
diff --git a/docs/tech.v3.dataset.clipboard.html b/docs/tech.v3.dataset.clipboard.html
diff --git a/docs/tech.v3.dataset.column-filters.html b/docs/tech.v3.dataset.column-filters.html
diff --git a/docs/tech.v3.dataset.column.html b/docs/tech.v3.dataset.column.html
diff --git a/docs/tech.v3.dataset.html b/docs/tech.v3.dataset.html
diff --git a/docs/tech.v3.dataset.io.csv.html b/docs/tech.v3.dataset.io.csv.html
diff --git a/docs/tech.v3.dataset.io.datetime.html b/docs/tech.v3.dataset.io.datetime.html
diff --git a/docs/tech.v3.dataset.io.string-row-parser.html b/docs/tech.v3.dataset.io.string-row-parser.html
diff --git a/docs/tech.v3.dataset.io.univocity.html b/docs/tech.v3.dataset.io.univocity.html
diff --git a/docs/tech.v3.dataset.join.html b/docs/tech.v3.dataset.join.html
diff --git a/docs/tech.v3.dataset.math.html b/docs/tech.v3.dataset.math.html
diff --git a/docs/tech.v3.dataset.metamorph.html b/docs/tech.v3.dataset.metamorph.html
diff --git a/docs/tech.v3.dataset.modelling.html b/docs/tech.v3.dataset.modelling.html
diff --git a/docs/tech.v3.dataset.print.html b/docs/tech.v3.dataset.print.html
diff --git a/docs/tech.v3.dataset.reductions.apache-data-sketch.html b/docs/tech.v3.dataset.reductions.apache-data-sketch.html
diff --git a/docs/tech.v3.dataset.reductions.html b/docs/tech.v3.dataset.reductions.html
diff --git a/docs/tech.v3.dataset.rolling.html b/docs/tech.v3.dataset.rolling.html
diff --git a/docs/tech.v3.dataset.set.html b/docs/tech.v3.dataset.set.html
diff --git a/docs/tech.v3.dataset.tensor.html b/docs/tech.v3.dataset.tensor.html
diff --git a/docs/tech.v3.dataset.zip.html b/docs/tech.v3.dataset.zip.html
diff --git a/docs/tech.v3.libs.arrow.html b/docs/tech.v3.libs.arrow.html
diff --git a/docs/tech.v3.libs.clj-transit.html b/docs/tech.v3.libs.clj-transit.html
diff --git a/docs/tech.v3.libs.fastexcel.html b/docs/tech.v3.libs.fastexcel.html
diff --git a/docs/tech.v3.libs.guava.cache.html b/docs/tech.v3.libs.guava.cache.html
diff --git a/docs/tech.v3.libs.parquet.html b/docs/tech.v3.libs.parquet.html
diff --git a/docs/tech.v3.libs.poi.html b/docs/tech.v3.libs.poi.html
diff --git a/docs/tech.v3.libs.smile.data.html b/docs/tech.v3.libs.smile.data.html
diff --git a/docs/tech.v3.libs.tribuo.html b/docs/tech.v3.libs.tribuo.html
diff --git a/scripts/enable-jdk21-m1 b/scripts/enable-jdk21-m1
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+
+VERSION="21.0.2"
+
+if [ ! -e jdk-$VERSION ]; then
+   echo "Downloading JDK $VERSION"	
+    wget https://download.java.net/java/GA/jdk21.0.2/f2283984656d49d69e91c558476027ac/13/GPL/openjdk-21.0.2_macos-aarch64_bin.tar.gz
+    tar -xvzf openjdk-21.0.2_macos-aarch64_bin.tar.gz
+    rm openjdk-21.0.2_macos-aarch64_bin.tar.gz
+fi
+
+export PATH=$(pwd)/jdk-$VERSION/bin:$PATH
+export JAVA_HOME=$(pwd)/jdk-$VERSION/
diff --git a/src/tech/v3/libs/clj_transit.clj b/src/tech/v3/libs/clj_transit.clj
@@ -140,8 +140,8 @@
        (text-col->data col)
        (#{:packed-local-date :local-date} col-dt)
        (obj-col->numeric-b64 col :int32 dtype-dt/local-date->days-since-epoch)
-       (#{:packed-instant :instant} col-dt)
-       (obj-col->numeric-b64 col :int64 dtype-dt/instant->microseconds-since-epoch)
+       (#{:packed-instant :instant :packed-milli-instant} col-dt)
+       (obj-col->numeric-b64 col :int64 dtype-dt/instant->milliseconds-since-epoch)
        :else ;;Punt!!
        (vec col))}))
 
@@ -238,7 +238,7 @@
                                     (= :instant dtype)
                                     (-> (b64->numeric-data data :int64)
                                         (dtype/->array-buffer)
-                                        (abuf/set-datatype :packed-instant))
+                                        (abuf/set-datatype :packed-milli-instant))
                                     :else
                                     (dtype/make-container dtype data))
                                  :name            (:name metadata)})))
@@ -260,12 +260,12 @@
 (def ^{:doc "Transit write handlers for java.time.LocalDate and java.time.Instant"}
   java-time-write-handlers
   {LocalDate (t/write-handler "java.time.LocalDate" dtype-dt/local-date->days-since-epoch)
-   Instant (t/write-handler "java.time.Instant" dtype-dt/instant->microseconds-since-epoch)})
+   Instant (t/write-handler "java.time.Instant" dtype-dt/instant->milliseconds-since-epoch)})
 
 (def ^{:doc "Transit read handlers for java.time.LocalDate and java.time.Instant"}
   java-time-read-handlers
   {"java.time.LocalDate" (t/read-handler dtype-dt/days-since-epoch->local-date)
-   "java.time.Instant" (t/read-handler dtype-dt/microseconds-since-epoch->instant)})
+   "java.time.Instant" (t/read-handler dtype-dt/milliseconds-since-epoch->instant)})
 
 
 (defn dataset->transit

diff --git a/test/tech/v3/dataset/parse_test.clj b/test/tech/v3/dataset/parse_test.clj
@@ -505,7 +505,8 @@
 (deftest issue-434-transit-support
   (let [ds (ds/->dataset {:a [1 2 3]
                           :b [:one :two :three]
-                          :c [(java.time.Instant/now) (java.time.Instant/now)]})
+                          ;;transit encoding is milli instants
+                          :c (dtype/make-container :packed-milli-instant [(java.time.Instant/now) (java.time.Instant/now)])})
         str-data (ds-transit/dataset->transit-str ds)
         nds (ds-transit/transit-str->dataset str-data)]
     (is (= (ds :a) (nds :a)))