Data Science Tutorial for Beginners

(use 'clojure.tools.deps.alpha.repl)
(clojure-version)

ls | grep csv

(slurp "combats.csv")

(require '[clojure.data.csv :as csv]
         '[clojure.java.io :as io]
         '[fastmath.stats :as stats])

(defn csv-reader
  [filename]
  (with-open [reader (io/reader filename)]
    (doall
      (csv/read-csv reader))))

(defn strip [coll chars]
  (apply str (remove #((set chars) %) coll)))

(defn data->maps
  [data]
  (doall
  (map zipmap
       (->> (first data)
         		(map #(strip % " .,"))
            (map keyword)
            repeat)
	  (rest data))))

(defn csv->maps
  [filename]
  (data->maps (csv-reader filename)))

(def pokemon (csv->maps "pokemon.csv"))

(first pokemon)

(def pokemon 
  (let [nums (map #(select-keys % [:SpDef :Attack 
                                   :Speed :SpAtk 
                                   :# :Defense 
                                   :Generation :HP]) pokemon)
        upvals (fn [m] 
                 (reduce-kv 
                  (fn [m k v] 
                    (assoc m k (Integer/parseInt v))) 
                  {} m))
        ->nums (map upvals nums)]
    (map #(merge %1 %2) pokemon ->nums)))

(first pokemon)

(defn nilify
  [v]
  (if (string? v)
    (when-not (clojure.string/blank? v)
      v)
    v))

(defn upvals [m f]
    (reduce-kv 
      (fn [m k v] 
        (assoc m k (f v))) 
      {} m))

(def pokemon (map (fn [m] (upvals m nilify)) pokemon))

(require '[huri.core :as h])

(h/size pokemon)

(h/cols pokemon)

(defn nilcounter
  [df]
  (let [c (h/cols df)
        dists (map #(h/distribution % df) c)
        nils  (map #(% nil) dists)]
    (zipmap c nils)))

(clojure.pprint/print-table [(nilcounter pokemon)])

(require '[clojure.math.combinatorics :as combinatorics])

(defn cross-corr
  [df numcols]
  (let [pairs (combinatorics/cartesian-product numcols numcols)]
    (for [p pairs]
      (if (apply = p)
        {:keys (vec p) :corr 1}
        {:keys (vec p) :corr (->> df (map #(select-keys % p)) 
                     h/col-oriented vals (apply stats/correlation))}))))

(defn heatmap
  [x y z]
  {:data 
    [{:x x
      :y y
      :z z
      :type "heatmap"
      :colorscale [[0 "#24ff24"] [1 "#004b00"]]}]
   :layout {:autosize false :width 600 :height 600}})

(def numcols [:SpDef :Attack :Speed :SpAtk 
              :# :Defense :Generation :HP])

(heatmap numcols 
         numcols
         (->> (cross-corr pokemon numcols)
             h/col-oriented
             :corr
             (partition 8)))

(.plot js/Nextjournal plotly↩)

{:data 
    [{:x (map :# pokemon)
      :y (map :Speed pokemon)
      :type "scatter"
      :name "Speed"
      :opacity 0.6}
     {:x (map :# pokemon)
      :y (map :Defense pokemon)
      :type "scatter"
      :name "Defense"
      :opacity 0.6}]
   :layout {:autosize false :width 600 :height 600}}

(.plot js/Nextjournal speed-defense↩)

(defn scatter
  [df kx ky]
  {:data 
    [{:x (map kx df)
      :y (map ky df)
      :mode "markers"
      :type "scatter"}]
   :layout {:autosize false :width 600 :height 600}})

(scatter pokemon :Attack :Defense)

(.plot js/Nextjournal attkdfs↩)

(defn hist
  [df kx]
  {:data 
    [{:x (map kx df)
      :type "histogram"}]
   :layout {:autosize false :width 600 :height 600}})

(hist pokemon :Speed)

(.plot js/Nextjournal histspeed↩)

(h/rollup :Type1 count pokemon)

(let [df (h/rollup :Legendary identity :Attack pokemon)]
  {:data 
    [{:y (df "False")
      :type "box"
      :name "False"}
     {:y (df "True")
      :type "box"
      :name "True"}]
   :layout {:autosize false :width 600 :height 600}})

(.plot js/Nextjournal box↩)

(map type (vals (first pokemon)))

(group-by :Name (map #(select-keys % [:Name :Attack :Defense]) pokemon))

(def pokemon-clean
  (filter #(not (nil? (:Type2 %))) pokemon))

(h/distribution :Type2 pokemon-clean)

(defn fill-nil
  [row k sub]
  (if (nil? (k row))
    (assoc row k sub)
    row))

(def pokemon (map #(fill-nil % :Type2 "empty") pokemon))

(let [n (map :# pokemon)]
  {:data 
    [{:y (map :Attack pokemon)
      :x n
      :type "scatter"
      :name "Attack"}
     {:y (map :Defense pokemon)
      :x n
      :type "scatter"
      :name "Defense"
      :xaxis "x2"
      :yaxis "y2"}
     {:y (map :Speed pokemon)
      :x n
      :type "scatter"
      :name "Speed"
      :xaxis "x3"
      :yaxis "y3"}]
   :layout {:autosize false :width 600 :height 600
            :grid {:rows 3 :columns 1 
                   :pattern "independent"
                   :roworder "bottom to top"}}})

(.plot js/Nextjournal multiline↩)

(assoc-in (hist pokemon :Speed) [:data 0 :cumulative] {:enabled true})

(.plot js/Nextjournal histcum↩)

(->> pokemon first :HP)