transpose-by-key

Author: cnuernber

src/tech/v3/dataset_api.clj

@@ -601,7 +601,7 @@ test/data/stocks.csv [10 3]:
   [col missing scalar-val]
   (let [missing (bitmap/->bitmap missing)]
     (if (.isEmpty missing)
-      col      
+      col
       (let [cbuf (dtype/->buffer (ds-proto/column-buffer col))
             col-dt (dtype/elemwise-datatype col)
             ec (.lsize cbuf)]
@@ -1469,6 +1469,27 @@ _unnamed [4 5]:
                                                 (column-names final-ds))))))))))
 
 
+(defn transpose-by-key
+  "Transposes a dataset by turning the values in column `k` into new column headers.
+
+  The remaining columns are rotated such that their original names become a
+  metadata column (defaulting to `:col`).
+
+  Arguments:
+  * `ds`: The source dataset (tech.ml.dataset).
+  * `k`: The column key whose row values will become the new column names.
+  * `options`: (Optional) A map where `:column-name` specifies the name of the
+    newly created column containing the old header names. Defaults to `:col`.
+
+  Returns a new dataset where rows have been flipped to columns based on `k`."
+  ([ds k] (transpose-by-key ds k nil))
+  ([ds k options]
+   (column ds k)
+   (let [sub-ds (dissoc ds k)]
+     (->> (lznc/map hamf/vector (column ds k) (rowvecs sub-ds))
+          (into {(get options :column-name :col) (column-names sub-ds)})
+          (->>dataset)))))
+
 
 (comment
   (export-symbols/write-api! 'tech.v3.dataset-api

src/tech/v3/libs/arrow/jpnz_lz4.clj

@@ -0,0 +1,39 @@
+(ns tech.v3.libs.arrow.jpnz-lz4
+  (:require [tech.v3.datatype :as dtype]
+            [clojure.tools.logging :as log])
+  (:import [tech.v3.datatype.array_buffer ArrayBuffer]
+           [java.io ByteArrayOutputStream ByteArrayInputStream]))
+
+(defn- ensure-bytes-array-buffer
+  ^ArrayBuffer [data]
+  (if-let [ary-buf (dtype/as-array-buffer data)]
+    (if (= :int8 (dtype/elemwise-datatype ary-buf))
+      ary-buf
+      (dtype/make-container :int8 data))))
+
+(defn create-jpnz-lz4-frame-compressor
+  [comp-map]
+  (assert (= :lz4 (get comp-map :compression-type)))
+  (fn [compbuf dstbuf]
+    (let [^ByteArrayOutputStream dstbuf (or dstbuf (java.io.ByteArrayOutputStream.))
+          os (net.jpountz.lz4.LZ4FrameOutputStream. dstbuf)
+          srcbuf (ensure-bytes-array-buffer compbuf)
+          ^bytes src-data (.ary-data srcbuf)]
+      (.write os src-data (unchecked-int (.offset srcbuf)) (unchecked-int (.n-elems srcbuf)))
+      (.close os)
+      (let [final-bytes (.toByteArray dstbuf)]
+        (.reset dstbuf)
+        {:writer-cache dstbuf
+         :dst-buffer final-bytes}))))
+
+(defn create-jpnz-lz4-decompressor
+  []
+  (log/warn "Unable to load native lz4 library, falling back to jpountz.
+Dependent block frames are not supported!!")
+  (fn [srcbuf dstbuf]
+    (let [src-byte-data (dtype/->byte-array srcbuf)
+          bis (ByteArrayInputStream. src-byte-data)
+          is (net.jpountz.lz4.LZ4FrameInputStream. bis)
+          temp-dstbuf (byte-array (dtype/ecount dstbuf))]
+      (.read is temp-dstbuf)
+      (dtype/copy! temp-dstbuf dstbuf))))