potential pathway of research - incomplete

Author: cnuernber

src/tech/v3/dataset/impl/column.clj

@@ -41,7 +41,7 @@
 
 
 (defn- reduce-column-buffer
-  [rfn acc src missing primitive-missing-value sidx eidx]
+  [rfn acc src missing primitive-missing-value missing-value sidx eidx]
   (let [sidx (long sidx)
         eidx (long eidx)
         ^RoaringBitmap missing missing
@@ -52,10 +52,10 @@
                                     (RoaringBitmap/and missing (bitmap/->bitmap sidx eidx))))
         ^java.util.function.LongBinaryOperator next-missing
         (hamf-fn/long-binary-operator
-         sidx next-missing-idx
-         (if (== sidx next-missing-idx)
-           (long (if (.hasNext int-iter) (Integer/toUnsignedLong (.next int-iter)) -1))
-           next-missing-idx))
+            sidx next-missing-idx
+            (if (== sidx next-missing-idx)
+              (long (if (.hasNext int-iter) (Integer/toUnsignedLong (.next int-iter)) -1))
+              next-missing-idx))
         missing-idx (.applyAsLong next-missing 0 0)]
     (cond
       (instance? IFn$OLO rfn)
@@ -90,15 +90,15 @@
              missing-idx missing-idx]
         (if (< sidx eidx)
           (let [acc (rfn acc (if (== sidx missing-idx)
-                               nil
+                               missing-value
                                (.readObject src sidx)))]
             (if (reduced? acc)
               (deref acc)
               (recur (unchecked-inc sidx) acc (.applyAsLong next-missing sidx missing-idx))))
           acc)))))
 
 (defn- kv-reduce-column-buffer
-  [rfn acc src missing primitive-missing-value sidx eidx]
+  [rfn acc src missing primitive-missing-value missing-value op-dtype sidx eidx]
   (let [sidx (long sidx)
         eidx (long eidx)
         ^RoaringBitmap missing missing
@@ -130,7 +130,8 @@
 (defn ^:no-doc make-column-buffer
   (^Buffer [^RoaringBitmap missing data dtype op-dtype]
    (let [^Buffer src (dtype-proto/->buffer data)
-         missing-value (column-base/datatype->missing-value dtype)
+         missing-value (when (identical? dtype op-dtype)
+                         (column-base/datatype->missing-value dtype))
          primitive-missing-value (column-base/datatype->packed-missing-value dtype)]
      ;;Sometimes we can utilize a pure passthrough.
      (if (.isEmpty missing)
@@ -170,7 +171,8 @@
                  (readObject [rdr idx] (.readObject this (+ idx sidx)))
                  (reduce [this rfn acc]
                    (reduce-column-buffer rfn acc src missing
-                                         primitive-missing-value sidx eidx))
+                                         primitive-missing-value missing-value
+                                         sidx eidx))
                  (kvreduce [this rfn acc]
                    (kv-reduce-column-buffer rfn acc src missing primitive-missing-value sidx eidx))))))
          (readLong [this idx]
@@ -182,7 +184,8 @@
              Double/NaN
              (.readDouble src idx)))
          (readObject [this idx]
-           (when-not (.contains missing idx)
+           (if (.contains missing idx)
+             missing-value
              (.readObject src idx)))
          (writeLong [this idx val]
            (.remove missing (unchecked-int idx))
@@ -197,7 +200,7 @@
                (.writeObject src idx val))
              (.add missing (unchecked-int idx))))
          (reduce [this rfn acc]
-           (reduce-column-buffer rfn acc src missing primitive-missing-value
+           (reduce-column-buffer rfn acc src missing primitive-missing-value missing-value
                                  0 (.lsize src)))
          (kvreduce [this rfn acc]
            (kv-reduce-column-buffer rfn acc src missing primitive-missing-value

src/tech/v3/dataset/impl/dataset.clj

@@ -315,9 +315,12 @@
       retval
       (throw (RuntimeException. (str "Column not found: " cname)))))
 
-  (rowvecs [ds options]
+  (rowvecs [ds {:keys [datatype] :as options}]
     (let [readers (hamf/object-array-list
-                   (lznc/map dtype/->reader columns))
+                   (lznc/map (if datatype
+                               #(dtype-proto/elemwise-reader-cast % datatype)
+                               dtype/->reader)
+                             columns))
           n-cols (count readers)
           n-rows (long (ds-proto/row-count ds))
           copying? (get options :copying? true)
@@ -335,24 +338,24 @@
               8 (row-vec-copying 8)
               (let [crange (hamf/range n-cols)]
                 (hamf-fn/long->obj
-                 row-idx
-                 (->> crange
-                      (lznc/map (hamf-fn/long->obj
-                                 col-idx (.readObject ^Buffer (.get readers col-idx) row-idx)))
-                      (hamf/vec)))))
+                    row-idx
+                    (->> crange
+                         (lznc/map (hamf-fn/long->obj
+                                       col-idx (.readObject ^Buffer (.get readers col-idx) row-idx)))
+                         (hamf/vec)))))
             ;;Non-copying in-place reader
             (let [crange (hamf/range n-cols)]
               (hamf-fn/long->obj
-               row-idx
-               (reify ObjectReader
-                 (lsize [this] n-cols)
-                 (readObject [this col-idx]
-                   (.readObject ^Buffer (.get readers col-idx) row-idx))
-                 (reduce [this rfn acc]
-                   (reduce (hamf-rf/long-accumulator
-                            acc col-idx (rfn acc ((.get readers col-idx) row-idx)))
-                           acc
-                           crange))))))]
+                  row-idx
+                  (reify ObjectReader
+                    (lsize [this] n-cols)
+                    (readObject [this col-idx]
+                      (.readObject ^Buffer (.get readers col-idx) row-idx))
+                    (reduce [this rfn acc]
+                      (reduce (hamf-rf/long-accumulator
+                                  acc col-idx (rfn acc ((.get readers col-idx) row-idx)))
+                              acc
+                              crange))))))]
       (reify ObjectReader
         (lsize [rdr] n-rows)
         (readObject [rdr row-idx] (.invokePrim row-fn row-idx))
@@ -361,8 +364,8 @@
               (ds-proto/rowvecs options)))
         (reduce [rdr rfn acc]
           (reduce (hamf-rf/long-accumulator
-                   acc row-idx
-                   (rfn acc (.invokePrim row-fn row-idx)))
+                      acc row-idx
+                      (rfn acc (.invokePrim row-fn row-idx)))
                   acc (hamf/range n-rows))))))
 
 

test/tech/v3/dataset_test.clj

@@ -1299,7 +1299,7 @@
   (let [src-ds (ds/->dataset {:a [1 2 nil 4]})
         dst-ds (assoc src-ds :b (dfn/+ (:a src-ds ) 1))]
     (is (= 1 (dtype/ecount (ds/missing (dst-ds :b)))))
-    (is (= [2.0 3.0 nil 5.0]
+    (is (= [2.0 3.0 ##NaN 5.0]
            (vec (dst-ds :b))))))
 
 
@@ -1869,6 +1869,13 @@
            (vec (ds/rows (ds/head (ds/column-map df2 :d (fn ^long [a b c] 1) [:a :b :c]) 5)))))
     (is (= (vec (repeat 5 {:a 1 :b 2 :c 3 :d 1.0}))
            (vec (ds/rows (ds/head (ds/column-map df2 :d (fn ^double [a b c] 1.0) [:a :b :c]) 5)))))))
+
+(deftest tc-issue-200
+  (let [ds (ds/->dataset {:a [0 Double/NaN 2]})
+        ]))
+
+
+
 (comment
   (require '[criterium.core :as crit])
   (def data (vec (repeatedly 100000 (fn [] {:a (rand-int 20) :b (rand) :c (rand)}))))