mirrors/pgloader
1
0
Fork 0
mirror of https://github.com/dimitri/pgloader.git synced 2026-05-05 10:56:10 +02:00
Code Issues Projects Releases Wiki Activity

Refactor the SQLite source files.

Browse Source
This commit is contained in:
Dimitri Fontaine 2014-11-09 22:59:30 +01:00
parent 5bc5a52582
commit ca325ba799
4 changed files with 165 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pgloader.asd
View File

@ -97,9 +97,17 @@
(:file "fixed")
(:file "db3")
(:file "ixf")
(:file "sqlite")
(:file "syslog")
(:module "sqlite-utils"
:pathname "sqlite"
:components
((:file "sqlite-cast-rules")
(:file "sqlite-schema"
:depends-on ("sqlite-cast-rules"))))
(:file "sqlite" :depends-on ("sqlite-utils"))
(:module "mysql-utils"
:pathname "mysql"
:components

148
src/sources/sqlite.lisp
View File

@ -4,154 +4,6 @@
(in-package :pgloader.sqlite)
(defvar *sqlite-db* nil
"The SQLite database connection handler.")
(defparameter *sqlite-default-cast-rules*
`((:source (:type "character") :target (:type "text" :drop-typemod t))
(:source (:type "varchar") :target (:type "text" :drop-typemod t))
(:source (:type "nvarchar") :target (:type "text" :drop-typemod t))
(:source (:type "char") :target (:type "text" :drop-typemod t))
(:source (:type "nchar") :target (:type "text" :drop-typemod t))
(:source (:type "clob") :target (:type "text" :drop-typemod t))
(:source (:type "tinyint") :target (:type "smallint"))
(:source (:type "float") :target (:type "float")
:using pgloader.transforms::float-to-string)
(:source (:type "real") :target (:type "real")
:using pgloader.transforms::float-to-string)
(:source (:type "double") :target (:type "double precision")
:using pgloader.transforms::float-to-string)
(:source (:type "numeric") :target (:type "numeric")
:using pgloader.transforms::float-to-string)
(:source (:type "blob") :target (:type "bytea")
:using pgloader.transforms::byte-vector-to-bytea)
(:source (:type "datetime") :target (:type "timestamptz")
:using pgloader.transforms::sqlite-timestamp-to-timestamp)
(:source (:type "timestamp") :target (:type "timestamp")
:using pgloader.transforms::sqlite-timestamp-to-timestamp)
(:source (:type "timestamptz") :target (:type "timestamptz")
:using pgloader.transforms::sqlite-timestamp-to-timestamp))
"Data Type Casting to migrate from SQLite to PostgreSQL")
;;;
;;; SQLite tools connecting to a database
;;;
(defstruct (coldef
(:constructor make-coldef (table-name
seq name dtype ctype
nullable default pk-id)))
table-name seq name dtype ctype nullable default pk-id)
(defun normalize (sqlite-type-name)
"SQLite only has a notion of what MySQL calls column_type, or ctype in the
CAST machinery. Transform it to the data_type, or dtype."
(let* ((sqlite-type-name (string-downcase sqlite-type-name))
(tokens (remove-if (lambda (token)
(member token '("unsigned" "short"
"varying" "native")
:test #'string-equal))
(sq:split-sequence #\Space sqlite-type-name))))
(assert (= 1 (length tokens)))
(first tokens)))
(defun ctype-to-dtype (sqlite-type-name)
"In SQLite we only get the ctype, e.g. int(7), but here we want the base
data type behind it, e.g. int."
(let* ((ctype (normalize sqlite-type-name))
(paren-pos (position #\( ctype)))
(if paren-pos (subseq ctype 0 paren-pos) ctype)))
(defun cast-sqlite-column-definition-to-pgsql (sqlite-column)
"Return the PostgreSQL column definition from the MySQL one."
(multiple-value-bind (column fn)
(with-slots (table-name name dtype ctype default nullable)
sqlite-column
(cast table-name name dtype ctype default nullable nil))
;; the SQLite driver smartly maps data to the proper CL type, but the
;; pgloader API only wants to see text representations to send down the
;; COPY protocol.
(values column (or fn (lambda (val) (if val (format nil "~a" val) :null))))))
(defmethod cast-to-bytea-p ((col coldef))
"Returns a generalized boolean, non-nil when the column is casted to a
PostgreSQL bytea column."
(string= "bytea" (cast-sqlite-column-definition-to-pgsql col)))
(defmethod format-pgsql-column ((col coldef) &key identifier-case)
"Return a string representing the PostgreSQL column definition."
(let* ((column-name
(apply-identifier-case (coldef-name col) identifier-case))
(type-definition
(with-slots (table-name name dtype ctype nullable default)
col
(cast table-name name dtype ctype default nullable nil))))
(format nil "~a ~22t ~a" column-name type-definition)))
(defun list-tables (&optional (db *sqlite-db*))
"Return the list of tables found in SQLITE-DB."
(let ((sql "SELECT tbl_name
FROM sqlite_master
WHERE type='table' AND tbl_name <> 'sqlite_sequence'"))
(loop for (name) in (sqlite:execute-to-list db sql)
collect name)))
(defun list-columns (table-name &optional (db *sqlite-db*))
"Return the list of columns found in TABLE-NAME."
(let ((sql (format nil "PRAGMA table_info(~a)" table-name)))
(loop for (seq name type nullable default pk-id) in
(sqlite:execute-to-list db sql)
collect (make-coldef table-name
seq
name
(ctype-to-dtype (normalize type))
(normalize type)
(= 1 nullable)
(unquote default)
pk-id))))
(defun list-all-columns (&optional (db *sqlite-db*))
"Get the list of SQLite column definitions per table."
(loop for table-name in (list-tables db)
collect (cons table-name (list-columns table-name db))))
(defstruct sqlite-idx name table-name sql)
(defmethod index-table-name ((index sqlite-idx))
(sqlite-idx-table-name index))
(defmethod format-pgsql-create-index ((index sqlite-idx) &key identifier-case)
"Generate the PostgresQL statement to build the given SQLite index definition."
(declare (ignore identifier-case))
(sqlite-idx-sql index))
(defun list-all-indexes (&optional (db *sqlite-db*))
"Get the list of SQLite index definitions per table."
(let ((sql "SELECT name, tbl_name, replace(replace(sql, '[', ''), ']', '')
FROM sqlite_master
WHERE type='index'"))
(loop :with schema := nil
:for (index-name table-name sql) :in (sqlite:execute-to-list db sql)
:when sql
:do (let ((entry (assoc table-name schema :test 'equal))
(idxdef (make-sqlite-idx :name index-name
:table-name table-name
:sql sql)))
(if entry
(push idxdef (cdr entry))
(push (cons table-name (list idxdef)) schema)))
:finally (return (reverse (loop for (name . indexes) in schema
collect (cons name (reverse indexes))))))))
;;;
;;; Integration with the pgloader Source API
;;;

94
src/sources/sqlite/sqlite-cast-rules.lisp Normal file
View File

@ -0,0 +1,94 @@
;;;
;;; Tools to handle the SQLite Database
;;;
(in-package :pgloader.sqlite)
(defvar *sqlite-db* nil
"The SQLite database connection handler.")
(defparameter *sqlite-default-cast-rules*
`((:source (:type "character") :target (:type "text" :drop-typemod t))
(:source (:type "varchar") :target (:type "text" :drop-typemod t))
(:source (:type "nvarchar") :target (:type "text" :drop-typemod t))
(:source (:type "char") :target (:type "text" :drop-typemod t))
(:source (:type "nchar") :target (:type "text" :drop-typemod t))
(:source (:type "clob") :target (:type "text" :drop-typemod t))
(:source (:type "tinyint") :target (:type "smallint"))
(:source (:type "float") :target (:type "float")
:using pgloader.transforms::float-to-string)
(:source (:type "real") :target (:type "real")
:using pgloader.transforms::float-to-string)
(:source (:type "double") :target (:type "double precision")
:using pgloader.transforms::float-to-string)
(:source (:type "numeric") :target (:type "numeric")
:using pgloader.transforms::float-to-string)
(:source (:type "blob") :target (:type "bytea")
:using pgloader.transforms::byte-vector-to-bytea)
(:source (:type "datetime") :target (:type "timestamptz")
:using pgloader.transforms::sqlite-timestamp-to-timestamp)
(:source (:type "timestamp") :target (:type "timestamp")
:using pgloader.transforms::sqlite-timestamp-to-timestamp)
(:source (:type "timestamptz") :target (:type "timestamptz")
:using pgloader.transforms::sqlite-timestamp-to-timestamp))
"Data Type Casting to migrate from SQLite to PostgreSQL")
(defstruct (coldef
(:constructor make-coldef (table-name
seq name dtype ctype
nullable default pk-id)))
table-name seq name dtype ctype nullable default pk-id)
(defun normalize (sqlite-type-name)
"SQLite only has a notion of what MySQL calls column_type, or ctype in the
CAST machinery. Transform it to the data_type, or dtype."
(let* ((sqlite-type-name (string-downcase sqlite-type-name))
(tokens (remove-if (lambda (token)
(member token '("unsigned" "short"
"varying" "native")
:test #'string-equal))
(sq:split-sequence #\Space sqlite-type-name))))
(assert (= 1 (length tokens)))
(first tokens)))
(defun ctype-to-dtype (sqlite-type-name)
"In SQLite we only get the ctype, e.g. int(7), but here we want the base
data type behind it, e.g. int."
(let* ((ctype (normalize sqlite-type-name))
(paren-pos (position #\( ctype)))
(if paren-pos (subseq ctype 0 paren-pos) ctype)))
(defun cast-sqlite-column-definition-to-pgsql (sqlite-column)
"Return the PostgreSQL column definition from the MySQL one."
(multiple-value-bind (column fn)
(with-slots (table-name name dtype ctype default nullable)
sqlite-column
(cast table-name name dtype ctype default nullable nil))
;; the SQLite driver smartly maps data to the proper CL type, but the
;; pgloader API only wants to see text representations to send down the
;; COPY protocol.
(values column (or fn (lambda (val) (if val (format nil "~a" val) :null))))))
(defmethod cast-to-bytea-p ((col coldef))
"Returns a generalized boolean, non-nil when the column is casted to a
PostgreSQL bytea column."
(string= "bytea" (cast-sqlite-column-definition-to-pgsql col)))
(defmethod format-pgsql-column ((col coldef) &key identifier-case)
"Return a string representing the PostgreSQL column definition."
(let* ((column-name
(apply-identifier-case (coldef-name col) identifier-case))
(type-definition
(with-slots (table-name name dtype ctype nullable default)
col
(cast table-name name dtype ctype default nullable nil))))
(format nil "~a ~22t ~a" column-name type-definition)))

62
src/sources/sqlite/sqlite-schema.lisp Normal file
View File

@ -0,0 +1,62 @@
;;;
;;; SQLite tools connecting to a database
;;;
(in-package :pgloader.sqlite)
(defvar *sqlite-db* nil
"The SQLite database connection handler.")
(defun list-tables (&optional (db *sqlite-db*))
"Return the list of tables found in SQLITE-DB."
(let ((sql "SELECT tbl_name
FROM sqlite_master
WHERE type='table' AND tbl_name <> 'sqlite_sequence'"))
(loop for (name) in (sqlite:execute-to-list db sql)
collect name)))
(defun list-columns (table-name &optional (db *sqlite-db*))
"Return the list of columns found in TABLE-NAME."
(let ((sql (format nil "PRAGMA table_info(~a)" table-name)))
(loop for (seq name type nullable default pk-id) in
(sqlite:execute-to-list db sql)
collect (make-coldef table-name
seq
name
(ctype-to-dtype (normalize type))
(normalize type)
(= 1 nullable)
(unquote default)
pk-id))))
(defun list-all-columns (&optional (db *sqlite-db*))
"Get the list of SQLite column definitions per table."
(loop for table-name in (list-tables db)
collect (cons table-name (list-columns table-name db))))
(defstruct sqlite-idx name table-name sql)
(defmethod index-table-name ((index sqlite-idx))
(sqlite-idx-table-name index))
(defmethod format-pgsql-create-index ((index sqlite-idx) &key identifier-case)
"Generate the PostgresQL statement to build the given SQLite index definition."
(declare (ignore identifier-case))
(sqlite-idx-sql index))
(defun list-all-indexes (&optional (db *sqlite-db*))
"Get the list of SQLite index definitions per table."
(let ((sql "SELECT name, tbl_name, replace(replace(sql, '[', ''), ']', '')
FROM sqlite_master
WHERE type='index'"))
(loop :with schema := nil
:for (index-name table-name sql) :in (sqlite:execute-to-list db sql)
:when sql
:do (let ((entry (assoc table-name schema :test 'equal))
(idxdef (make-sqlite-idx :name index-name
:table-name table-name
:sql sql)))
(if entry
(push idxdef (cdr entry))
(push (cons table-name (list idxdef)) schema)))
:finally (return (reverse (loop for (name . indexes) in schema
collect (cons name (reverse indexes))))))))