fix(cli): rewrite 'doom sync'; deprecate 'doom build'

This changes 'doom sync' to be smarter about responding to changed
package recipes/pins, changes in Emacs version, or instances where the user
has copied a config to a new system.

In all these cases, the user would formerly have to know about a
specific combination of 'doom sync -u' and 'doom build' to ensure Doom
is in a good state. With this change, 'doom sync' handles all these
cases.

Also, 'doom build' is now deprecated (and 'doom sync' now has a
--rebuild option to mimic its old behavior).

Also also, sometimes, a package may silently fail when cloned (which
used to result in an empty repo). Now, if this is detected, cloning will
be re-attempted up to 3 times before aborting with much more visible
error.

Note: these are stopgap solutions, until v3 is finished.
This commit is contained in:
Henrik Lissner 2024-03-24 17:38:40 -04:00
parent 1fa8d3a4b9
commit cff091982e
No known key found for this signature in database
GPG Key ID: B60957CA074D39A3
6 changed files with 195 additions and 212 deletions

View File

@ -138,8 +138,6 @@ commands you should know about:
+ `doom env` to dump a snapshot of your shell environment to a file that Doom
will load at startup. This allows Emacs to inherit your `PATH`, among other
things.
+ `doom build` to recompile all installed packages (use this if you up/downgrade
Emacs).
# Roadmap

View File

@ -280,7 +280,6 @@ SEE ALSO:
;; (load! "nuke" dir)
;; (load! "package" dir)
;; (load! "profile" dir)
;; (defcli-obsolete! ((build b)) (sync "--rebuild") "v3.0.0")
)
(defcli-group! "Diagnostics"

View File

@ -402,14 +402,13 @@ This command is never needed for changes to =$DOOMDIR/config.el=.
** Copy or sync my config to another system?
*Short answer:* it is safe to sync =$DOOMDIR= across systems, but not
=$EMACSDIR=. Once moved, use ~$ doom sync && doom build~ to ensure everything is
set up correctly.
=$EMACSDIR=. Once moved, use ~$ doom sync~ to ensure everything is set up
correctly.
*Long answer:* packages can contain baked-in absolute paths and non-portable
byte-code. It is never a good idea to mirror it across multiple systems, unless
they are all the same (same OS, same version of Emacs, same paths). Most issues
should be solved by running ~$ doom sync && doom build~ on the other end, once
moved.
should be solved by running ~$ doom sync~ on the other end, once moved.
** Start over, in case something went terribly wrong?
Delete =$EMACSDIR/.local/straight= and run ~$ doom sync~.

View File

@ -13,25 +13,8 @@
;;
;;; Commands
(defcli! (:before (build b purge p)) (&context context)
(require 'comp nil t)
(doom-initialize-core-packages))
(defcli-obsolete! ((build b)) (sync "--rebuild") "v3.0.0")
;; DEPRECATED Replace with "doom sync --rebuild"
(defcli! ((build b))
((rebuild-p ("-r") "Only rebuild packages that need rebuilding")
(jobs ("-j" "--jobs" num) "How many CPUs to use for native compilation"))
"Byte-compiles & symlinks installed packages.
This ensures that all needed files are symlinked from their package repo and
their elisp files are byte-compiled. This is especially necessary if you upgrade
Emacs (as byte-code is generally not forward-compatible)."
:benchmark t
(when jobs
(setq native-comp-async-jobs-number (truncate jobs)))
(when (doom-packages-build (not rebuild-p))
(doom-profile-generate))
t)
;; TODO Rename to "doom gc" and move to its own file
(defcli! ((purge p))
@ -50,6 +33,8 @@ possible.
It is a good idea to occasionally run this doom purge -g to ensure your package
list remains lean."
:benchmark t
(require 'comp nil t)
(doom-initialize-core-packages)
(straight-check-all)
(when (doom-packages-purge
(not noelpa-p)
@ -242,170 +227,149 @@ list remains lean."
(defun doom-packages--write-missing-eln-errors ()
"Write .error files for any expected .eln files that are missing."
(when (featurep 'native-compile)
(cl-loop for file in doom-packages--eln-output-expected
for eln-name = (doom-packages--eln-file-name file)
for eln-file = (doom-packages--eln-output-file eln-name)
for error-file = (doom-packages--eln-error-file eln-name)
for error-dir = (file-name-directory error-file)
unless (or (file-exists-p eln-file)
(file-newer-than-file-p error-file file)
(not (file-writable-p error-dir)))
do (make-directory error-dir 'parents)
(write-region "" nil error-file)
(doom-log "Wrote %s" error-file))
(setq doom-packages--eln-output-expected nil)))
(cl-loop for file in doom-packages--eln-output-expected
for eln-name = (doom-packages--eln-file-name file)
for eln-file = (doom-packages--eln-output-file eln-name)
for error-file = (doom-packages--eln-error-file eln-name)
for error-dir = (file-name-directory error-file)
unless (or (file-exists-p eln-file)
(file-newer-than-file-p error-file file)
(not (file-writable-p error-dir)))
do (make-directory error-dir 'parents)
(write-region "" nil error-file)
(doom-log "Wrote %s" error-file))
(setq doom-packages--eln-output-expected nil))
(defun doom-packages--compile-site-files ()
"Queue async compilation for all non-doom Elisp files."
(when (featurep 'native-compile)
(cl-loop with paths = (cl-loop for path in load-path
unless (file-in-directory-p path doom-local-dir)
collect path)
for file in (doom-files-in paths :match "\\.el\\(?:\\.gz\\)?$")
if (and (file-exists-p (byte-compile-dest-file file))
(not (doom-packages--find-eln-file (doom-packages--eln-file-name file)))
(not (cl-some (fn! (string-match-p % file))
native-comp-deferred-compilation-deny-list))) do
(doom-log "Compiling %s" file)
(native-compile-async file))))
(cl-loop with paths = (cl-loop for path in load-path
unless (file-in-directory-p path doom-local-dir)
collect path)
for file in (doom-files-in paths :match "\\.el\\(?:\\.gz\\)?$")
if (and (file-exists-p (byte-compile-dest-file file))
(not (doom-packages--find-eln-file (doom-packages--eln-file-name file)))
(not (cl-some (fn! (string-match-p % file))
native-comp-deferred-compilation-deny-list))) do
(doom-log "Compiling %s" file)
(native-compile-async file)))
(defun doom-packages-install ()
"Installs missing packages.
This function will install any primary package (i.e. a package with a `package!'
declaration) or dependency thereof that hasn't already been."
(defun doom-packages-ensure (&optional force-p)
"Ensure packages are installed, built"
(doom-initialize-packages)
(print! (start "Installing packages..."))
(let ((pinned (doom-package-pinned-list)))
(print-group!
(add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h)
(if-let (built
(doom-packages--with-recipes (doom-package-recipe-list)
(recipe package type local-repo)
(let ((repo-dir (straight--repos-dir local-repo)))
(unless (file-directory-p repo-dir)
(doom-packages--cli-recipes-update))
(condition-case-unless-debug e
(let ((straight-use-package-pre-build-functions
(cons (lambda (pkg &rest _)
(when-let (commit (cdr (assoc pkg pinned)))
(print! (item "Checked out %s: %s") pkg commit)))
straight-use-package-pre-build-functions)))
;; HACK: Straight can sometimes fail to clone a repo,
;; leaving behind an empty directory which, in future
;; invocations, it will assume indicates a successful
;; clone (causing load errors later).
(let ((try 0))
(while (or (not (file-directory-p repo-dir))
(directory-empty-p repo-dir))
(if (= try 3)
(error "Failed to clone package")
(when (> try 0)
(print! "Failed to clone %S, trying again (attempt #%d)..." package (1+ try))))
(delete-file (file-name-concat (straight--modified-dir) package))
(delete-directory repo-dir t)
(delete-directory (straight--build-dir package) t)
(straight-use-package (intern package))
(cl-incf try)))
;; HACK: Line encoding issues can plague repos with
;; dirty worktree prompts when updating packages or
;; "Local variables entry is missing the suffix"
;; errors when installing them (see #2637), so have
;; git handle conversion by force.
(when (and doom--system-windows-p (stringp local-repo))
(let ((default-directory (straight--repos-dir local-repo)))
(when (file-in-directory-p default-directory straight-base-dir)
(straight--process-run "git" "config" "core.autocrlf" "true")))))
(error
(signal 'doom-package-error (list package e)))))))
(progn
(when (featurep 'native-compile)
(doom-packages--compile-site-files)
(doom-packages--wait-for-native-compile-jobs)
(doom-packages--write-missing-eln-errors))
(print! (success "\033[KInstalled %d packages") (length built)))
(print! (item "No packages need to be installed"))
nil))))
(defun doom-packages-build (&optional force-p)
"(Re)build all packages."
(doom-initialize-packages)
(print! (start "(Re)building %spackages...") (if force-p "all " ""))
(if (not (file-directory-p (straight--repos-dir)))
(print! (start "Installing all packages for the first time (this may take a while)..."))
(if force-p
(print! (start "Rebuilding all packages (this may take a while)..."))
(print! (start "Ensuring packages are installed and built..."))))
(print-group!
(let ((straight-check-for-modifications
(when (file-directory-p (straight--modified-dir))
'(find-when-checking)))
(straight--allow-find
(and straight-check-for-modifications
(executable-find straight-find-executable)
t))
(straight--packages-not-to-rebuild
(or straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
(straight--packages-to-rebuild
(or (if force-p :all straight--packages-to-rebuild)
(make-hash-table :test #'equal)))
(recipes (doom-package-recipe-list)))
(add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h)
(unless force-p
(straight--make-build-cache-available))
(if-let (built
(doom-packages--with-recipes recipes (package local-repo recipe)
(unless force-p
;; Ensure packages with outdated files/bytecode are rebuilt
(let* ((build-dir (straight--build-dir package))
(repo-dir (straight--repos-dir local-repo))
(build (if (plist-member recipe :build)
(plist-get recipe :build)
t))
(want-byte-compile
(or (eq build t)
(memq 'compile build)))
(want-native-compile
(or (eq build t)
(memq 'native-compile build))))
(and (eq (car-safe build) :not)
(setq want-byte-compile (not want-byte-compile)
want-native-compile (not want-native-compile)))
(unless (featurep 'native-compile)
(setq want-native-compile nil))
(and (or want-byte-compile want-native-compile)
(or (file-newer-than-file-p repo-dir build-dir)
(file-exists-p (straight--modified-dir (or local-repo package)))
(cl-loop with outdated = nil
for file in (doom-files-in build-dir :match "\\.el$" :full t)
if (or (if want-byte-compile (doom-packages--elc-file-outdated-p file))
(if want-native-compile (doom-packages--eln-file-outdated-p file)))
do (setq outdated t)
(let ((straight-check-for-modifications
(when (file-directory-p (straight--modified-dir))
'(find-when-checking)))
(straight--allow-find
(and straight-check-for-modifications
(executable-find straight-find-executable)
t))
(straight--packages-not-to-rebuild
(or straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
(straight--packages-to-rebuild
(or (if force-p :all straight--packages-to-rebuild)
(make-hash-table :test #'equal)))
(recipes (doom-package-recipe-list))
(pinned (doom-package-pinned-list)))
(add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h)
(straight--make-build-cache-available)
(if-let (built
(doom-packages--with-recipes recipes (package local-repo recipe)
(let ((repo-dir (straight--repos-dir (or local-repo package)))
(build-dir (straight--build-dir package)))
(unless force-p
;; Ensure packages with outdated files/bytecode are rebuilt
(let* ((build (if (plist-member recipe :build)
(plist-get recipe :build)
t))
(want-byte-compile
(or (eq build t)
(memq 'compile build)))
(want-native-compile
(or (eq build t)
(memq 'native-compile build))))
(and (eq (car-safe build) :not)
(setq want-byte-compile (not want-byte-compile)
want-native-compile (not want-native-compile)))
(unless (featurep 'native-compile)
(setq want-native-compile nil))
(and (or want-byte-compile want-native-compile)
(or (file-newer-than-file-p repo-dir build-dir)
(file-exists-p (straight--modified-dir (or local-repo package)))
(cl-loop with outdated = nil
for file in (doom-files-in build-dir :match "\\.el$" :full t)
if (or (if want-byte-compile (doom-packages--elc-file-outdated-p file))
(if want-native-compile (doom-packages--eln-file-outdated-p file)))
do (setq outdated t)
(when want-native-compile
(push file doom-packages--eln-output-expected))
finally return outdated))
(puthash package t straight--packages-to-rebuild))))
(straight-use-package (intern package))))
(progn
(when (featurep 'native-compile)
(doom-packages--compile-site-files)
(doom-packages--wait-for-native-compile-jobs)
(doom-packages--write-missing-eln-errors))
;; HACK Every time you save a file in a package that straight tracks,
;; it is recorded in ~/.emacs.d/.local/straight/modified/.
;; Typically, straight will clean these up after rebuilding, but
;; Doom's use-case circumnavigates that, leaving these files
;; there and causing a rebuild of those packages each time `doom
;; sync' or similar is run, so we clean it up ourselves:
(delete-directory (straight--modified-dir) 'recursive)
(print! (success "\033[KRebuilt %d package(s)") (length built)))
(print! (item "No packages need rebuilding"))
nil))))
finally return outdated))
(puthash package t straight--packages-to-rebuild))))
(unless (file-directory-p repo-dir)
(doom-packages--cli-recipes-update))
(condition-case-unless-debug e
(let ((straight-vc-git-post-clone-hook
(cons (lambda! (&key repo-dir commit)
(print-group!
(if-let (pin (cdr (assoc package pinned)))
(print! (item "Pinned to %s") pin)
(print! (item "Checked out %s") commit)))
;; HACK: Line encoding issues can plague
;; repos with dirty worktree prompts
;; when updating packages or "Local
;; variables entry is missing the
;; suffix" errors when installing them
;; (see #2637), so have git handle
;; conversion by force.
(when (and doom--system-windows-p (stringp repo-dir))
(let ((default-directory repo-dir))
(when (file-in-directory-p default-directory straight-base-dir)
(straight--process-run "git" "config" "core.autocrlf" "true")))))
straight-vc-git-post-clone-hook)))
(straight-use-package (intern package))
;; HACK: Straight can sometimes fail to clone a repo,
;; leaving behind an empty directory which, in future
;; invocations, it will assume indicates a successful
;; clone (causing load errors later).
(let ((try 0))
(while (or (not (file-directory-p repo-dir))
(directory-empty-p repo-dir))
(when (= try 3)
(error "Failed to clone package"))
(print! "Failed to clone %S, trying again (attempt #%d)..." package (1+ try))
(delete-directory repo-dir t)
(delete-directory build-dir t)
(straight-use-package (intern package))
(cl-incf try))))
(error
(signal 'doom-package-error (list package e)))))))
(progn
(when (featurep 'native-compile)
(doom-packages--compile-site-files)
(doom-packages--wait-for-native-compile-jobs)
(doom-packages--write-missing-eln-errors))
;; HACK: Every time you save a file in a package that straight
;; tracks, it is recorded in ~/.emacs.d/.local/straight/modified/.
;; Typically, straight will clean these up after rebuilding, but
;; Doom's use-case circumnavigates that, leaving these files there
;; and causing a rebuild of those packages each time `doom sync'
;; or similar is run, so we clean it up ourselves:
(delete-directory (straight--modified-dir) 'recursive)
(print! (success "\033[KBuilt %d package(s)") (length built)))
(print! (item "No packages need attention"))
nil))))
(defun doom-packages-update ()
(defun doom-packages-update (&optional pinned-only-p)
"Updates packages."
(doom-initialize-packages)
(doom-packages--barf-if-incomplete)
(doom-packages--cli-recipes-update)
(let* ((repo-dir (straight--repos-dir))
(pinned (doom-package-pinned-list))
(recipes (doom-package-recipe-list))
@ -413,9 +377,10 @@ declaration) or dependency thereof that hasn't already been."
(repos-to-rebuild (make-hash-table :test 'equal))
(total (length recipes))
(esc (unless init-file-debug "\033[1A"))
(i 0)
errors)
(print! (start "Updating packages (this may take a while)..."))
(i 0))
(if pinned-only-p
(print! (start "Updating pinned packages..."))
(print! (start "Updating all packages (this may take a while)...")))
(doom-packages--with-recipes recipes (recipe package type local-repo)
(cl-incf i)
(print-group!
@ -428,11 +393,13 @@ declaration) or dependency thereof that hasn't already been."
(cl-return))
(let ((default-directory (straight--repos-dir local-repo)))
(unless (file-in-directory-p default-directory repo-dir)
(print! (warn "(%d/%d) Skipping %s because it is local") i total package)
(print! (warn "(%d/%d) Skipping %s because it is out-of-tree...") i total package)
(cl-return))
(when (eq type 'git)
(unless (file-exists-p ".git")
(error "%S is not a valid repository" package)))
(when (and pinned-only-p (not (assoc local-repo pinned)))
(cl-return))
(condition-case-unless-debug e
(let ((ref (straight-vc-get-commit type local-repo))
(target-ref
@ -446,13 +413,6 @@ declaration) or dependency thereof that hasn't already been."
(doom-packages--straight-with (straight-vc-fetch-from-remote recipe)
(when .it
(straight-merge-package package)
;; (condition-case e
;; (straight-merge-package package)
;; (wrong-type-argument
;; (if (not (equal (cdr e) '(arrayp nil)))
;; (signal (car e) (cdr e))
;; (delete-directory (straight--build-dir local-repo) t)
;; (straight-use-package (intern package)))))
(setq target-ref (straight-vc-get-commit type local-repo))
(setq output (doom-packages--commit-log-between ref target-ref)
commits (length (split-string output "\n" t)))
@ -480,7 +440,7 @@ declaration) or dependency thereof that hasn't already been."
(straight-vc-git-default-clone-depth 'full))
(delete-directory repo 'recursive)
(print-group!
(straight-use-package (intern package) nil 'no-build))
(straight-use-package (intern package) nil 'no-build))
(prog1 (file-directory-p repo)
(or (not (eq type 'git))
(setq output (doom-packages--commit-log-between ref target-ref)
@ -536,13 +496,14 @@ declaration) or dependency thereof that hasn't already been."
(princ "\033[K")
(if (hash-table-empty-p packages-to-rebuild)
(ignore (print! (success "All %d packages are up-to-date") total))
(doom-packages--cli-recipes-update)
(straight--transaction-finalize)
(let ((default-directory (straight--build-dir)))
(mapc (doom-rpartial #'delete-directory 'recursive)
(hash-table-keys packages-to-rebuild)))
(print! (success "Updated %d package(s)")
(hash-table-count packages-to-rebuild))
(doom-packages-build)
(doom-packages-ensure)
t))))

View File

@ -14,18 +14,21 @@
(defvar doom-before-sync-hook ()
"Hooks run before 'doom sync' synchronizes the user's config with Doom.")
(defvar doom-cli-sync-info-file (file-name-concat doom-profile-data-dir "sync"))
;;
;;; Commands
(defcli-alias! (:before (sync s)) (:before build))
(defcli! ((sync s))
((noenvvar? ("-e") "Don't regenerate the envvar file")
(noelc? ("-c") "Don't recompile config")
(update? ("-u") "Update installed packages after syncing")
(update? ("-u") "Update all installed packages after syncing")
(noupdate? ("-U") "Don't update any packages")
(purge? ("-p") "Purge orphaned package repos & regraft them")
(jobs ("-j" "--jobs" num) "How many CPUs to use for native compilation"))
(jobs ("-j" "--jobs" num) "How many threads to use for native compilation")
(rebuild? ("-b" "--rebuild") "Rebuild, compile, & symlink installed packages")
(auto? ("-B") "Rebuild packages, but only if necessary")
&context context)
"Synchronize your config with Doom Emacs.
This is the equivalent of running autoremove, install, autoloads, then
@ -33,8 +36,10 @@ recompile. Run this whenever you:
1. Modify your `doom!' block,
2. Add or remove `package!' blocks to your config,
3. Add or remove autoloaded functions in module autoloaded files.
4. Update Doom outside of Doom (e.g. with git)
3. Add or remove autoloaded functions in module autoloaded files,
4. Update Doom outside of Doom (e.g. with git),
5. Move your Doom config (either $EMACSDIR or $DOOMDIR) to a new location.
6. When you up (or down) grade Emacs itself.
It will ensure that unneeded packages are removed, all needed packages are
installed, autoloads files are up-to-date and no byte-compiled files have gone
@ -47,26 +52,47 @@ OPTIONS:
:benchmark t
(when (doom-profiles-bootloadable-p)
(call! '(profiles sync "--reload")))
(run-hooks 'doom-before-sync-hook)
(add-hook 'kill-emacs-hook #'doom-sync--abort-warning-h)
(when (doom-cli-context-suppress-prompts-p context)
(setq auto? t))
(when jobs
(setq native-comp-async-jobs-number (truncate jobs)))
(print! (start "Synchronizing %S profile..." )
(or (car doom-profile) "default"))
(run-hooks 'doom-before-sync-hook)
(add-hook 'kill-emacs-hook #'doom-sync--abort-warning-h)
(print! (item "Using Emacs %s @ %s") emacs-version (path invocation-directory invocation-name))
(print! (start "Synchronizing %S profile..." ) (or (car doom-profile) "default"))
(unwind-protect
(print-group!
(when (and (not noenvvar?)
(file-exists-p doom-env-file))
(call! '(env)))
(doom-packages-install)
(doom-packages-build)
(when update?
(doom-packages-update))
(doom-packages-purge purge? purge? purge? purge? purge?)
(when (doom-profile-generate)
(print! (item "Restart Emacs or use 'M-x doom/reload' for changes to take effect"))
(run-hooks 'doom-after-sync-hook))
t)
;; If the user has up/downgraded Emacs since last sync, or copied their
;; config to a different system, then their packages need to be
;; recompiled. This is necessary because Emacs byte-code is not
;; necessarily back/forward compatible across major versions, and many
;; packages bake in hardcoded data at compile-time.
(pcase-let ((`(,old-version . ,old-host) (doom-file-read doom-cli-sync-info-file :by 'read :noerror t))
(to-rebuild nil))
(when (and old-version (not (equal old-version emacs-version)))
(print! (warn "Emacs version has changed since last sync (from %s to %s)") old-version emacs-version)
(setq to-rebuild t))
(when (and old-host (not (equal old-host (system-name))))
(print! (warn "Your system has changed since last sync"))
(setq to-rebuild t))
(when (and to-rebuild (not auto?))
(or (y-or-n-p
(format! " %s" "Your installed packages will need to be recompiled. Do so now?"))
(exit! 0))
(setq rebuild? t)))
(when (and (not noenvvar?)
(file-exists-p
(file-name-concat doom-profile-dir
doom-profile-env-file-name)))
(call! '(env)))
(doom-packages-ensure rebuild?)
(unless noupdate? (doom-packages-update (not update?)))
(doom-packages-purge purge? purge? purge? purge? purge?)
(when (doom-profile-generate)
(print! (item "Restart Emacs or use 'M-x doom/reload' for changes to take effect"))
(run-hooks 'doom-after-sync-hook))
(with-temp-file doom-cli-sync-info-file (prin1 (cons emacs-version (system-name)) (current-buffer)))
t)
(remove-hook 'kill-emacs-hook #'doom-sync--abort-warning-h)))

View File

@ -32,7 +32,7 @@ following shell commands:
doom clean
doom sync -u"
(let* ((force? (doom-cli-context-suppress-prompts-p context))
(sync-cmd (append '("sync" "-u") (if jobs `("-j" ,num)))))
(sync-cmd (append '("sync" "-u" "-B") (if jobs `("-j" ,num)))))
(cond
(packages?
;; HACK It's messy to use straight to upgrade straight, due to the