X-Git-Url: https://iankelling.org/git/?p=bbdb-csv-import;a=blobdiff_plain;f=bbdb-csv-import.el;h=b234bd6066eeae900d9e2caa123b4a53f75dabbb;hp=512a94871e5451e7ec5f76f0b545838280df2c0c;hb=HEAD;hpb=edf99589ac4aabc8940af78d4a90a8ec35d3f5d7 diff --git a/bbdb-csv-import.el b/bbdb-csv-import.el index 512a948..b234bd6 100644 --- a/bbdb-csv-import.el +++ b/bbdb-csv-import.el @@ -9,6 +9,7 @@ ;; Package-Requires: ((pcsv "1.3.3") (dash "2.5.0") (bbdb "20140412.1949")) ;; Keywords: csv, util, bbdb ;; Homepage: https://gitlab.com/iankelling/bbdb-csv-import +;; Mailing-List: https://lists.iankelling.org/listinfo/bbdb-csv-import ;; This program is free software; you can redistribute it and/or modify ;; it under the terms of the GNU General Public License as published by @@ -24,7 +25,7 @@ ;; along with this program. If not, see . ;;; Commentary: - +;; ;; Importer of csv (comma separated value) text into Emacs’s bbdb database, ;; version 3+. Works out of the box with csv exported from Thunderbird, Gmail, ;; Linkedin, Outlook.com/hotmail, and probably others. @@ -41,29 +42,41 @@ ;; ;;; Basic Usage: ;; -;; You may want to back up existing data in ~/.bbdb and ~/.emacs.d/bbdb in case -;; you don't like the newly imported data. +;; Back up bbdb by copying `bbdb-file' in case things go wrong. ;; ;; Simply M-x `bbdb-csv-import-buffer' or `bbdb-csv-import-file'. ;; When called interactively, they prompt for file or buffer arguments. +;; +;; Then view your bbdb records: M-x bbdb .* RET +;; If the import looks good save the bbdb database: C-x s (bbdb-save) ;;; Advanced usage / notes: -;; Tested to work with thunderbird, gmail, linkedin, outlook.com/hotmail.com For -;; those programs, if it's exporter has an option of what kind of csv format, -;; choose it's own native format if available, if not, choose an outlook -;; compatible format. If you're exporting from some other program and its csv -;; exporter claims outlook compatibility, there is a good chance it will work -;; out of the box. ;; -;; If things don't work, you can probably fix it with a custom field mapping -;; variable. It should not be too hard. Use the existing tables as an -;; example. By default, we use a combination of most predefined mappings, and -;; look for all of their fields, but it is probably best to avoid that kind of -;; table when setting up your own as it is an unnecessary complexity in that -;; case. If you have a problem with data from a supported export program, start -;; by testing its specific mapping table instead of the combined one. Here is a -;; handy template to set each of the predefined mapping tables if you would -;; rather avoid the configure interface: +;; Tested to work with thunderbird, gmail, linkedin, +;; outlook.com/hotmail.com. For those programs, if it's exporter has an option +;; of what kind of csv format, choose it's own native format if available, if +;; not, choose an outlook compatible format. If you're exporting from some other +;; program and its csv exporter claims outlook compatibility, there is a good +;; chance it will work out of the box. If it doesn't, you can try to fix it as +;; described below, or the maintainer will be happy to help, just anonymize your +;; csv data using the M-x bbdb-csv-anonymize-current-buffer (make sure csv +;; buffer is the current one) and attach it to an email to the mailing list. +;; +;; Duplicate contacts (according to email address) are skipped if +;; bbdb-allow-duplicates is nil (default). Any duplicates found are echoed at +;; the end of the import. + +;;; Custom mapping of csv fields +;; +;; If a field is handled wrong or you want to extend the program to handle a new +;; kind of csv format, you need to setup a custom field mapping variable. Use +;; the existing tables as an example. By default, we use a combination of most +;; predefined mappings, and look for all of their fields, but it is probably +;; best to avoid that kind of table when setting up your own as it is an +;; unnecessary complexity in that case. If you have a problem with data from a +;; supported export program, start by testing its specific mapping table instead +;; of the combined one. Here is a handy template to set each of the predefined +;; mapping tables if you would rather avoid the configure interface: ;; ;; (setq bbdb-csv-import-mapping-table bbdb-csv-import-combined) ;; (setq bbdb-csv-import-mapping-table bbdb-csv-import-thunderbird) @@ -73,20 +86,47 @@ ;; (setq bbdb-csv-import-mapping-table bbdb-csv-import-outlook-web) ;; (setq bbdb-csv-import-mapping-table bbdb-csv-import-outlook-typed-email) ;; -;; In addition to the examples, the doc string for `bbdb-create-internal' may -;; also be useful. Please send any new mapping tables to the maintainer listed -;; in this file. The maintainer should be able to help with any issues and may -;; create a new mapping table given sample data. +;; The doc string for `bbdb-create-internal' may also be useful when creating a +;; mapping table. If you create a table for a program not not already supported, +;; please share it with the mailing list so it can be added to this program. +;; The maintainer should be able to help with any issues and may create a new +;; mapping table given sample data. +;; +;; Mapping table tips: +;; * The repeat keyword expands numbered field names, based on the first +;; field, as many times as they exist in the csv data. +;; * All mapping fields are optional. A simple mapping table could be +;; (setq bbdb-csv-import-mapping-table '((:mail "Primary Email"))) +;; * :xfields uses the csv field name to create custom fields in bbdb. It downcases +;; the field name, and replaces spaces with "-", and repeating dashes with a +;; single one . For example, if you had a csv named "Mail Alias" or "Mail - alias", +;; you could add it to :xfields in a mapping table and it would become "mail-alias" +;; in bbdb. + +;;; Misc tips/troubleshooting: ;; -;; Misc tips/troubleshooting: ;; - ASynK looks promising for syncing bbdb/google/outlook. ;; - The git repo contains a test folder with exactly tested version info and working -;; test data. +;; test data. Software, and especially online services are prone to changing how they +;; export. Please send feedback if you run into problems. ;; - bbdb doesn't work if you delete the bbdb database file in ;; the middle of an emacs session. If you want to empty the current bbdb database, ;; do M-x bbdb then .* then C-u * d on the beginning of a record. ;; - After changing a mapping table variable, don't forget to re-execute ;; (setq bbdb-csv-import-mapping-table ...) so that it propagates. +;; - :namelist is used instead of :name if 2 or more non-empty fields from :namelist are +;; found in a record. If :name is empty, we try a single non-empty field from :namelist +;; This sounds a bit strange, but it's to try and deal with Thunderbird idiosyncrasies. + +;;; Bugs, patches, discussion, feedback +;; +;; Patches and bugs are very welcome via https://gitlab.com/iankelling/bbdb-csv-import +;; +;; Questions, feedback, or anything is very welcome at to the bbdb-csv-import mailing list +;; https://lists.iankelling.org/listinfo/bbdb-csv-import, no subscription needed to post via +;; bbdb-csv-import@lists.iankelling.org. The maintainer would probably be happy +;; to work on new features if something is missing. + ;;; Code: @@ -95,7 +135,6 @@ (require 'bbdb-com) (eval-when-compile (require 'cl)) - (defconst bbdb-csv-import-thunderbird '((:namelist "First Name" "Last Name") (:name "Display Name") @@ -300,10 +339,13 @@ See the commentary section of this file for more details." (defun bbdb-csv-import-expand-repeats (csv-fields list) - "Return new list where elements from LIST in form (repeat elem1 ...) -become ((elem1 ...) [(elem2 ...)] ...) for as many repeating -numbered fields exist in the csv fields. elem can be a string or -a tree (a list with lists inside it)" + "Return new list where elements from LIST in form (repeat elem1 +...) become ((elem1 ...) [(elem2 ...)] ...) for as many fields +exist in the csv fields. elem can be a string or a tree (a list +with lists inside it). We use the first element as a template, +and increase its number by one, and check if it exists, and then +increment any other elements from the repeat list which have +numbers in them." (cl-flet ((replace-num (num string) ;; in STRING, replace all groups of numbers with NUM (replace-regexp-in-string "[0-9]+" @@ -314,7 +356,7 @@ a tree (a list with lists inside it)" (cons it acc) (setq it (cdr it)) (let* ((i 1) - (first-field (car (flatten it)))) + (first-field (car (-flatten it)))) (setq acc (cons it acc)) ;; use first-field to test if there is another repetition. (while (member @@ -343,28 +385,39 @@ don't want flattened." ;;;###autoload (defun bbdb-csv-import-file (filename) - "Parse and import csv file FILENAME to bbdb." + "Parse and import csv file FILENAME to bbdb. +The file will be saved to disk with blank lines and aberrant characters removed." (interactive "fCSV file containg contact data: ") (bbdb-csv-import-buffer (find-file-noselect filename))) ;;;###autoload (defun bbdb-csv-import-buffer (&optional buffer-or-name) - "Parse and import csv BUFFER-OR-NAME to bbdb. -Argument is a buffer or name of a buffer. -Defaults to current buffer." + "Parse and import csv buffer to bbdb. Interactively, it prompts for a buffer. +The buffer will be saved to disk with blank lines and aberrant characters removed. +BUFFER-OR-NAME is a buffer or name of a buffer, or the current buffer if nil." (interactive "bBuffer containing CSV contact data: ") (when (null bbdb-csv-import-mapping-table) (error "error: `bbdb-csv-import-mapping-table' is nil. Please set it and rerun.")) - (let* ((csv-data (pcsv-parse-buffer (get-buffer (or buffer-or-name (current-buffer))))) + (let* ((csv-buffer (get-buffer (or buffer-or-name (current-buffer)))) + (csv-data (save-excursion + (set-buffer csv-buffer) + ;; deal with blank lines and ^M from linkedin + (flush-lines "^\\s-*$") + (goto-char (point-min)) + ;; remove ^M aka ret characters + (while (re-search-forward (char-to-string 13) nil t) + (replace-match "")) + (basic-save-buffer) + (pcsv-parse-file buffer-file-name))) (csv-fields (car csv-data)) (csv-data (cdr csv-data)) - (initial-duplicate-value bbdb-allow-duplicates) - csv-record rd assoc-plus map-bbdb) + (allow-dupes bbdb-allow-duplicates) + csv-record rd assoc-plus map-bbdb dupes) ;; convenient function names (fset 'rd 'bbdb-csv-import-rd) (fset 'assoc-plus 'bbdb-csv-import-assoc-plus) (fset 'map-bbdb (-partial 'bbdb-csv-import-map-bbdb csv-fields)) - ;; better to allow duplicates rather than fail + ;; we handle duplicates ourselves (setq bbdb-allow-duplicates t) ;; loop over the csv records (while (setq csv-record (map 'list 'cons csv-fields (pop csv-data))) @@ -380,11 +433,16 @@ Defaults to current buffer." (data (assoc-plus (if (consp e) (cadr e) e) csv-record))) (if data (list data-name data))))) ;; set the arguments to bbdb-create-internal, then call it, the end. - (let ((name (let ((name (rd-assoc :namelist))) - ;; prioritize any combination of first middle last over :name - (if (>= (length name) 2) - (mapconcat 'identity name " ") - (car (rd-assoc :name))))) + (let ((name (let ((namelist (rd-assoc :namelist)) + (let-name (car (rd-assoc :name)))) + ;; priority: 2 or more from :namelist, then non-empty :name, then + ;; any single element of :namelist + (cond ((>= (length namelist) 2) + (mapconcat 'identity namelist " ")) + ((not (null let-name)) + let-name) + (t + (mapconcat 'identity namelist " "))))) (affix (rd-assoc :affix)) (aka (rd-assoc :aka)) (organization (rd-assoc :organization)) @@ -420,15 +478,27 @@ Defaults to current buffer." (map-bbdb :address))) (xfields (rd (lambda (list) (let ((e (car list))) - (while (string-match "-" e) - (setq e (replace-match "" nil nil e))) (while (string-match " +" e) (setq e (replace-match "-" nil nil e))) + (while (string-match "--+" e) + (setq e (replace-match "-" nil nil e))) (setq e (make-symbol (downcase e))) (cons e (cadr list)))) ;; change from (a b) to (a . b) (rd #'assoc-expand (map-bbdb :xfields))))) - (bbdb-create-internal name affix aka organization mail phone address xfields t)))) - (setq bbdb-allow-duplicates initial-duplicate-value))) + ;; we copy and subvert bbdb's duplicate detection instead of catching + ;; errors so that we don't interfere with other errors, and can print + ;; them nicely at the end. + (let (found-dupe) + (dolist (elt mail) + (when (bbdb-gethash elt '(mail)) + (push elt dupes) + (setq found-dupe t))) + (when (or allow-dupes (not found-dupe)) + (bbdb-create-internal name affix aka organization mail phone address xfields t)))))) + (when dupes (if allow-dupes + (message "Warning, contacts with duplicate email addresses were imported:\n%s" dupes) + (message "Skipped contacts with duplicate email addresses:\n%s" dupes))) + (setq bbdb-allow-duplicates allow-dupes))) (defun bbdb-csv-import-rd (func list) "like mapcar but don't build nil results into the resulting list" @@ -444,6 +514,16 @@ Defaults to current buffer." (when (not (string= "" result)) result))) +(defun bbdb-csv-anonymize-current-buffer () + (interactive) + "Anonymize the current buffer which contains csv data. + The first line should contain header names." + (goto-line 2) + (while (re-search-forward "\\w") + (delete-char -1) + (insert (number-to-string (random 9))))) + + (provide 'bbdb-csv-import) ;;; bbdb-csv-import.el ends here