mirror of
https://github.com/dimitri/pgloader.git
synced 2026-02-10 17:01:02 +01:00
0549e74f6d
Experiment with the idea of splitting the read work in several concurrent threads, where each reader is reading portions of the target table, using a WHERE id <= x and id > y clause in its SELECT query. For this to kick-in a number of conditions needs to be met, as described in the documentation. The main interest might not be faster queries to overall fetch the same data set, but better concurrency with as many readers as writters and each couple its own dedicated queue.
97 lines
3.0 KiB
Common Lisp
97 lines
3.0 KiB
Common Lisp
;;;
|
|
;;; Random utilities
|
|
;;;
|
|
(in-package :pgloader.utils)
|
|
|
|
;;;
|
|
;;; Camel Case converter
|
|
;;;
|
|
(defun camelCase-to-colname (string)
|
|
"Transform input STRING into a suitable column name.
|
|
lahmanID lahman_id
|
|
playerID player_id
|
|
birthYear birth_year"
|
|
(coerce
|
|
(loop
|
|
for first = t then nil
|
|
for char across string
|
|
for previous-upper-p = nil then char-upper-p
|
|
for char-upper-p = (eq char (char-upcase char))
|
|
for new-word = (and (not first) char-upper-p (not previous-upper-p))
|
|
when (and new-word (not (char= char #\_))) collect #\_
|
|
collect (char-downcase char))
|
|
'string))
|
|
|
|
;;;
|
|
;;; Unquote SQLite default values, might be useful elsewhere
|
|
;;;
|
|
(defun unquote (string &optional (quote #\') (escape #\\))
|
|
"Given '0', returns 0."
|
|
(declare (type (or null simple-string) string))
|
|
(when string
|
|
(let ((l (length string)))
|
|
(cond ((and (<= 2 l) ; "string"
|
|
(char= quote (aref string 0) (aref string (1- l))))
|
|
(subseq string 1 (1- l)))
|
|
|
|
((and (<= 4 l) ; \"string\"
|
|
(char= escape (aref string 0) (aref string (- l 2)))
|
|
(char= quote (aref string 1) (aref string (- l 1))))
|
|
(subseq string 2 (- l 2)))
|
|
|
|
(t
|
|
string)))))
|
|
|
|
;;;
|
|
;;; Process ~/ references at run-time (not at compile time!)
|
|
;;;
|
|
(defun expand-user-homedir-pathname (namestring)
|
|
"Expand NAMESTRING replacing leading ~ with (user-homedir-pathname)"
|
|
(typecase namestring
|
|
(pathname namestring)
|
|
(string
|
|
(cond ((or (string= "~" namestring) (string= "~/" namestring))
|
|
(user-homedir-pathname))
|
|
|
|
((and (<= 2 (length namestring))
|
|
(char= #\~ (aref namestring 0))
|
|
(char= #\/ (aref namestring 1)))
|
|
(uiop:merge-pathnames*
|
|
(uiop:parse-unix-namestring (subseq namestring 2))
|
|
(user-homedir-pathname)))
|
|
|
|
(t
|
|
(uiop:parse-unix-namestring namestring))))))
|
|
|
|
;;;
|
|
;;; For log messages
|
|
;;;
|
|
(defun pretty-print-bytes (bytes &key (unit "B"))
|
|
"Return a string to reprensent bytes in human readable format, with units"
|
|
(let ((bytes (or bytes 0)))
|
|
(loop
|
|
:for multiple :in '("T" "G" "M" "k")
|
|
:for power :in '(40 30 20 10 1)
|
|
:for limit := (expt 2 power)
|
|
:until (<= limit bytes)
|
|
:finally (return
|
|
(format nil "~5,1f ~a~a" (/ bytes limit) multiple unit)))))
|
|
|
|
;;;
|
|
;;; Defining ranges and partitions.
|
|
;;;
|
|
(defun split-range (min max &optional (count *rows-per-range*))
|
|
"Split the range from MIN to MAX into sub-ranges of COUNT elements."
|
|
(loop :for i := min :then j
|
|
:for j := (+ i count)
|
|
:while (< i max)
|
|
:collect (list i (min j max))))
|
|
|
|
(defun distribute (list-of-ranges count)
|
|
"Split a list of ranges into COUNT sublists."
|
|
(let ((result (make-array count :element-type 'list :initial-element nil)))
|
|
(loop :for i :from 0
|
|
:for range :in list-of-ranges
|
|
:do (push range (aref result (mod i count))))
|
|
(map 'list #'reverse result )))
|