diff --git a/pgloader.asd b/pgloader.asd index 80ca1c1..7e7fdad 100644 --- a/pgloader.asd +++ b/pgloader.asd @@ -97,9 +97,17 @@ (:file "fixed") (:file "db3") (:file "ixf") - (:file "sqlite") (:file "syslog") + (:module "sqlite-utils" + :pathname "sqlite" + :components + ((:file "sqlite-cast-rules") + (:file "sqlite-schema" + :depends-on ("sqlite-cast-rules")))) + + (:file "sqlite" :depends-on ("sqlite-utils")) + (:module "mysql-utils" :pathname "mysql" :components diff --git a/src/sources/sqlite.lisp b/src/sources/sqlite.lisp index e1f2ac6..f3d57ec 100644 --- a/src/sources/sqlite.lisp +++ b/src/sources/sqlite.lisp @@ -4,154 +4,6 @@ (in-package :pgloader.sqlite) -(defvar *sqlite-db* nil - "The SQLite database connection handler.") - -(defparameter *sqlite-default-cast-rules* - `((:source (:type "character") :target (:type "text" :drop-typemod t)) - (:source (:type "varchar") :target (:type "text" :drop-typemod t)) - (:source (:type "nvarchar") :target (:type "text" :drop-typemod t)) - (:source (:type "char") :target (:type "text" :drop-typemod t)) - (:source (:type "nchar") :target (:type "text" :drop-typemod t)) - (:source (:type "clob") :target (:type "text" :drop-typemod t)) - - (:source (:type "tinyint") :target (:type "smallint")) - - (:source (:type "float") :target (:type "float") - :using pgloader.transforms::float-to-string) - - (:source (:type "real") :target (:type "real") - :using pgloader.transforms::float-to-string) - - (:source (:type "double") :target (:type "double precision") - :using pgloader.transforms::float-to-string) - - (:source (:type "numeric") :target (:type "numeric") - :using pgloader.transforms::float-to-string) - - (:source (:type "blob") :target (:type "bytea") - :using pgloader.transforms::byte-vector-to-bytea) - - (:source (:type "datetime") :target (:type "timestamptz") - :using pgloader.transforms::sqlite-timestamp-to-timestamp) - - (:source (:type "timestamp") :target (:type "timestamp") - :using pgloader.transforms::sqlite-timestamp-to-timestamp) - - (:source (:type "timestamptz") :target (:type "timestamptz") - :using pgloader.transforms::sqlite-timestamp-to-timestamp)) - "Data Type Casting to migrate from SQLite to PostgreSQL") - -;;; -;;; SQLite tools connecting to a database -;;; -(defstruct (coldef - (:constructor make-coldef (table-name - seq name dtype ctype - nullable default pk-id))) - table-name seq name dtype ctype nullable default pk-id) - -(defun normalize (sqlite-type-name) - "SQLite only has a notion of what MySQL calls column_type, or ctype in the - CAST machinery. Transform it to the data_type, or dtype." - (let* ((sqlite-type-name (string-downcase sqlite-type-name)) - (tokens (remove-if (lambda (token) - (member token '("unsigned" "short" - "varying" "native") - :test #'string-equal)) - (sq:split-sequence #\Space sqlite-type-name)))) - (assert (= 1 (length tokens))) - (first tokens))) - -(defun ctype-to-dtype (sqlite-type-name) - "In SQLite we only get the ctype, e.g. int(7), but here we want the base - data type behind it, e.g. int." - (let* ((ctype (normalize sqlite-type-name)) - (paren-pos (position #\( ctype))) - (if paren-pos (subseq ctype 0 paren-pos) ctype))) - -(defun cast-sqlite-column-definition-to-pgsql (sqlite-column) - "Return the PostgreSQL column definition from the MySQL one." - (multiple-value-bind (column fn) - (with-slots (table-name name dtype ctype default nullable) - sqlite-column - (cast table-name name dtype ctype default nullable nil)) - ;; the SQLite driver smartly maps data to the proper CL type, but the - ;; pgloader API only wants to see text representations to send down the - ;; COPY protocol. - (values column (or fn (lambda (val) (if val (format nil "~a" val) :null)))))) - -(defmethod cast-to-bytea-p ((col coldef)) - "Returns a generalized boolean, non-nil when the column is casted to a - PostgreSQL bytea column." - (string= "bytea" (cast-sqlite-column-definition-to-pgsql col))) - -(defmethod format-pgsql-column ((col coldef) &key identifier-case) - "Return a string representing the PostgreSQL column definition." - (let* ((column-name - (apply-identifier-case (coldef-name col) identifier-case)) - (type-definition - (with-slots (table-name name dtype ctype nullable default) - col - (cast table-name name dtype ctype default nullable nil)))) - (format nil "~a ~22t ~a" column-name type-definition))) - -(defun list-tables (&optional (db *sqlite-db*)) - "Return the list of tables found in SQLITE-DB." - (let ((sql "SELECT tbl_name - FROM sqlite_master - WHERE type='table' AND tbl_name <> 'sqlite_sequence'")) - (loop for (name) in (sqlite:execute-to-list db sql) - collect name))) - -(defun list-columns (table-name &optional (db *sqlite-db*)) - "Return the list of columns found in TABLE-NAME." - (let ((sql (format nil "PRAGMA table_info(~a)" table-name))) - (loop for (seq name type nullable default pk-id) in - (sqlite:execute-to-list db sql) - collect (make-coldef table-name - seq - name - (ctype-to-dtype (normalize type)) - (normalize type) - (= 1 nullable) - (unquote default) - pk-id)))) - -(defun list-all-columns (&optional (db *sqlite-db*)) - "Get the list of SQLite column definitions per table." - (loop for table-name in (list-tables db) - collect (cons table-name (list-columns table-name db)))) - -(defstruct sqlite-idx name table-name sql) - -(defmethod index-table-name ((index sqlite-idx)) - (sqlite-idx-table-name index)) - -(defmethod format-pgsql-create-index ((index sqlite-idx) &key identifier-case) - "Generate the PostgresQL statement to build the given SQLite index definition." - (declare (ignore identifier-case)) - (sqlite-idx-sql index)) - -(defun list-all-indexes (&optional (db *sqlite-db*)) - "Get the list of SQLite index definitions per table." - (let ((sql "SELECT name, tbl_name, replace(replace(sql, '[', ''), ']', '') - FROM sqlite_master - WHERE type='index'")) - (loop :with schema := nil - :for (index-name table-name sql) :in (sqlite:execute-to-list db sql) - :when sql - :do (let ((entry (assoc table-name schema :test 'equal)) - (idxdef (make-sqlite-idx :name index-name - :table-name table-name - :sql sql))) - (if entry - (push idxdef (cdr entry)) - (push (cons table-name (list idxdef)) schema))) - :finally (return (reverse (loop for (name . indexes) in schema - collect (cons name (reverse indexes)))))))) - - ;;; ;;; Integration with the pgloader Source API ;;; diff --git a/src/sources/sqlite/sqlite-cast-rules.lisp b/src/sources/sqlite/sqlite-cast-rules.lisp new file mode 100644 index 0000000..1070fd4 --- /dev/null +++ b/src/sources/sqlite/sqlite-cast-rules.lisp @@ -0,0 +1,94 @@ +;;; +;;; Tools to handle the SQLite Database +;;; + +(in-package :pgloader.sqlite) + +(defvar *sqlite-db* nil + "The SQLite database connection handler.") + +(defparameter *sqlite-default-cast-rules* + `((:source (:type "character") :target (:type "text" :drop-typemod t)) + (:source (:type "varchar") :target (:type "text" :drop-typemod t)) + (:source (:type "nvarchar") :target (:type "text" :drop-typemod t)) + (:source (:type "char") :target (:type "text" :drop-typemod t)) + (:source (:type "nchar") :target (:type "text" :drop-typemod t)) + (:source (:type "clob") :target (:type "text" :drop-typemod t)) + + (:source (:type "tinyint") :target (:type "smallint")) + + (:source (:type "float") :target (:type "float") + :using pgloader.transforms::float-to-string) + + (:source (:type "real") :target (:type "real") + :using pgloader.transforms::float-to-string) + + (:source (:type "double") :target (:type "double precision") + :using pgloader.transforms::float-to-string) + + (:source (:type "numeric") :target (:type "numeric") + :using pgloader.transforms::float-to-string) + + (:source (:type "blob") :target (:type "bytea") + :using pgloader.transforms::byte-vector-to-bytea) + + (:source (:type "datetime") :target (:type "timestamptz") + :using pgloader.transforms::sqlite-timestamp-to-timestamp) + + (:source (:type "timestamp") :target (:type "timestamp") + :using pgloader.transforms::sqlite-timestamp-to-timestamp) + + (:source (:type "timestamptz") :target (:type "timestamptz") + :using pgloader.transforms::sqlite-timestamp-to-timestamp)) + "Data Type Casting to migrate from SQLite to PostgreSQL") + +(defstruct (coldef + (:constructor make-coldef (table-name + seq name dtype ctype + nullable default pk-id))) + table-name seq name dtype ctype nullable default pk-id) + +(defun normalize (sqlite-type-name) + "SQLite only has a notion of what MySQL calls column_type, or ctype in the + CAST machinery. Transform it to the data_type, or dtype." + (let* ((sqlite-type-name (string-downcase sqlite-type-name)) + (tokens (remove-if (lambda (token) + (member token '("unsigned" "short" + "varying" "native") + :test #'string-equal)) + (sq:split-sequence #\Space sqlite-type-name)))) + (assert (= 1 (length tokens))) + (first tokens))) + +(defun ctype-to-dtype (sqlite-type-name) + "In SQLite we only get the ctype, e.g. int(7), but here we want the base + data type behind it, e.g. int." + (let* ((ctype (normalize sqlite-type-name)) + (paren-pos (position #\( ctype))) + (if paren-pos (subseq ctype 0 paren-pos) ctype))) + +(defun cast-sqlite-column-definition-to-pgsql (sqlite-column) + "Return the PostgreSQL column definition from the MySQL one." + (multiple-value-bind (column fn) + (with-slots (table-name name dtype ctype default nullable) + sqlite-column + (cast table-name name dtype ctype default nullable nil)) + ;; the SQLite driver smartly maps data to the proper CL type, but the + ;; pgloader API only wants to see text representations to send down the + ;; COPY protocol. + (values column (or fn (lambda (val) (if val (format nil "~a" val) :null)))))) + +(defmethod cast-to-bytea-p ((col coldef)) + "Returns a generalized boolean, non-nil when the column is casted to a + PostgreSQL bytea column." + (string= "bytea" (cast-sqlite-column-definition-to-pgsql col))) + +(defmethod format-pgsql-column ((col coldef) &key identifier-case) + "Return a string representing the PostgreSQL column definition." + (let* ((column-name + (apply-identifier-case (coldef-name col) identifier-case)) + (type-definition + (with-slots (table-name name dtype ctype nullable default) + col + (cast table-name name dtype ctype default nullable nil)))) + (format nil "~a ~22t ~a" column-name type-definition))) diff --git a/src/sources/sqlite/sqlite-schema.lisp b/src/sources/sqlite/sqlite-schema.lisp new file mode 100644 index 0000000..7be6cf1 --- /dev/null +++ b/src/sources/sqlite/sqlite-schema.lisp @@ -0,0 +1,62 @@ +;;; +;;; SQLite tools connecting to a database +;;; +(in-package :pgloader.sqlite) + +(defvar *sqlite-db* nil + "The SQLite database connection handler.") + +(defun list-tables (&optional (db *sqlite-db*)) + "Return the list of tables found in SQLITE-DB." + (let ((sql "SELECT tbl_name + FROM sqlite_master + WHERE type='table' AND tbl_name <> 'sqlite_sequence'")) + (loop for (name) in (sqlite:execute-to-list db sql) + collect name))) + +(defun list-columns (table-name &optional (db *sqlite-db*)) + "Return the list of columns found in TABLE-NAME." + (let ((sql (format nil "PRAGMA table_info(~a)" table-name))) + (loop for (seq name type nullable default pk-id) in + (sqlite:execute-to-list db sql) + collect (make-coldef table-name + seq + name + (ctype-to-dtype (normalize type)) + (normalize type) + (= 1 nullable) + (unquote default) + pk-id)))) + +(defun list-all-columns (&optional (db *sqlite-db*)) + "Get the list of SQLite column definitions per table." + (loop for table-name in (list-tables db) + collect (cons table-name (list-columns table-name db)))) + +(defstruct sqlite-idx name table-name sql) + +(defmethod index-table-name ((index sqlite-idx)) + (sqlite-idx-table-name index)) + +(defmethod format-pgsql-create-index ((index sqlite-idx) &key identifier-case) + "Generate the PostgresQL statement to build the given SQLite index definition." + (declare (ignore identifier-case)) + (sqlite-idx-sql index)) + +(defun list-all-indexes (&optional (db *sqlite-db*)) + "Get the list of SQLite index definitions per table." + (let ((sql "SELECT name, tbl_name, replace(replace(sql, '[', ''), ']', '') + FROM sqlite_master + WHERE type='index'")) + (loop :with schema := nil + :for (index-name table-name sql) :in (sqlite:execute-to-list db sql) + :when sql + :do (let ((entry (assoc table-name schema :test 'equal)) + (idxdef (make-sqlite-idx :name index-name + :table-name table-name + :sql sql))) + (if entry + (push idxdef (cdr entry)) + (push (cons table-name (list idxdef)) schema))) + :finally (return (reverse (loop for (name . indexes) in schema + collect (cons name (reverse indexes))))))))