Skip to content

Commit

Permalink
(apply concat nil) returns nil.
Browse files Browse the repository at this point in the history
  • Loading branch information
cnuernber committed Oct 29, 2021
1 parent 9ae8b13 commit 4a43d8b
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 44 deletions.
89 changes: 45 additions & 44 deletions src/tech/v3/dataset.cljs
Original file line number Diff line number Diff line change
Expand Up @@ -407,50 +407,51 @@ cljs.user> (->> (ds/->dataset {:a (range 100)
(defn concat
"This is a copying concatenation so the result will be realized. Missing columns
will be filled in with missing values."
[ds & args]
(if-not (seq args)
ds
(when-let [ds-list (->> (cljs.core/concat [ds] args)
(remove #(== 0 (row-count %)))
(seq))]
(if (== (count ds-list) 1)
(first ds-list)
(let [colnames (->> (mapcat column-names ds-list)
(distinct))
n-rows (apply + (map row-count ds-list))
col-dtypes (->> colnames
;;force missing column errors right here.
(map (fn [colname]
(->> ds-list
(map #(% colname))
(remove nil?)
(map dtype/elemwise-datatype)
(reduce (fn [lhs-dt rhs-dt]
(if (= lhs-dt rhs-dt)
lhs-dt
(casting/widest-datatype
lhs-dt rhs-dt))))))))]
(->> (map (fn [colname coldtype]
(let [container (col-impl/make-container coldtype)
missing (js/Set.)]
(dtype/ensure-capacity! container n-rows)
(doseq [ds ds-list]
(let [offset (count container)]
(if-let [ds-col (ds colname)]
(do
(dtype/iterate! #(.add missing (+ offset %))
(ds-proto/-missing ds-col))
(dtype/add-all! container ds-col))
(let [mv (col-impl/datatype->missing-value coldtype)]
;;they are all missing
(dotimes [idx (row-count ds)]
(.add missing (+ offset idx))
(dtype/add! container mv))))))
#:tech.v3.dataset{:data container
:missing missing
:name colname}))
colnames col-dtypes)
(ds-impl/new-dataset (meta ds))))))))
([ds & args]
(if-not (seq args)
ds
(when-let [ds-list (->> (cljs.core/concat [ds] args)
(remove #(== 0 (row-count %)))
(seq))]
(if (== (count ds-list) 1)
(first ds-list)
(let [colnames (->> (mapcat column-names ds-list)
(distinct))
n-rows (apply + (map row-count ds-list))
col-dtypes (->> colnames
;;force missing column errors right here.
(map (fn [colname]
(->> ds-list
(map #(% colname))
(remove nil?)
(map dtype/elemwise-datatype)
(reduce (fn [lhs-dt rhs-dt]
(if (= lhs-dt rhs-dt)
lhs-dt
(casting/widest-datatype
lhs-dt rhs-dt))))))))]
(->> (map (fn [colname coldtype]
(let [container (col-impl/make-container coldtype)
missing (js/Set.)]
(dtype/ensure-capacity! container n-rows)
(doseq [ds ds-list]
(let [offset (count container)]
(if-let [ds-col (ds colname)]
(do
(dtype/iterate! #(.add missing (+ offset %))
(ds-proto/-missing ds-col))
(dtype/add-all! container ds-col))
(let [mv (col-impl/datatype->missing-value coldtype)]
;;they are all missing
(dotimes [idx (row-count ds)]
(.add missing (+ offset idx))
(dtype/add! container mv))))))
#:tech.v3.dataset{:data container
:missing missing
:name colname}))
colnames col-dtypes)
(ds-impl/new-dataset (meta ds))))))))
([] nil))


(defn merge-by-column
Expand Down
4 changes: 4 additions & 0 deletions test/tech/v3/dataset_test.cljs
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,7 @@
(let [ds (-> (ds/->dataset {:a (range 10)})
(ds/filter-column :a #(> % 10)))]
(is (not (nil? (pr-str ds))))))


(deftest ds-concat-nil-seq
(is (nil? (apply ds/concat nil))))

0 comments on commit 4a43d8b

Please sign in to comment.