2 * Copyright 1998, 2000 Marshall Kirk McKusick.
3 * Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org>
6 * The soft updates code is derived from the appendix of a University
7 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt,
8 * "Soft Updates: A Solution to the Metadata Update Problem in File
9 * Systems", CSE-TR-254-95, August 1995).
11 * Further information about soft updates can be obtained from:
13 * Marshall Kirk McKusick http://www.mckusick.com/softdep/
14 * 1614 Oxford Street mckusick@mckusick.com
15 * Berkeley, CA 94709-1608 +1-510-843-9542
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions
22 * 1. Redistributions of source code must retain the above copyright
23 * notice, this list of conditions and the following disclaimer.
24 * 2. Redistributions in binary form must reproduce the above copyright
25 * notice, this list of conditions and the following disclaimer in the
26 * documentation and/or other materials provided with the distribution.
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
31 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
34 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 * from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00
42 #include <sys/cdefs.h>
43 __FBSDID("$FreeBSD$");
46 #include "opt_quota.h"
50 * For now we want the safety net that the DEBUG flag provides.
56 #include <sys/param.h>
57 #include <sys/kernel.h>
58 #include <sys/systm.h>
62 #include <sys/kthread.h>
64 #include <sys/limits.h>
66 #include <sys/malloc.h>
67 #include <sys/mount.h>
68 #include <sys/mutex.h>
69 #include <sys/namei.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/vnode.h>
78 #include <ufs/ufs/dir.h>
79 #include <ufs/ufs/extattr.h>
80 #include <ufs/ufs/quota.h>
81 #include <ufs/ufs/inode.h>
82 #include <ufs/ufs/ufsmount.h>
83 #include <ufs/ffs/fs.h>
84 #include <ufs/ffs/softdep.h>
85 #include <ufs/ffs/ffs_extern.h>
86 #include <ufs/ufs/ufs_extern.h>
89 #include <vm/vm_extern.h>
90 #include <vm/vm_object.h>
92 #include <geom/geom.h>
96 #define KTR_SUJ 0 /* Define to KTR_SPARE. */
101 softdep_flushfiles(oldmnt, flags, td)
102 struct mount *oldmnt;
107 panic("softdep_flushfiles called");
111 softdep_mount(devvp, mp, fs, cred)
129 softdep_uninitialize()
143 softdep_setup_sbupdate(ump, fs, bp)
144 struct ufsmount *ump;
151 softdep_setup_inomapdep(bp, ip, newinum, mode)
158 panic("softdep_setup_inomapdep called");
162 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags)
165 ufs2_daddr_t newblkno;
170 panic("softdep_setup_blkmapdep called");
174 softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
177 ufs2_daddr_t newblkno;
178 ufs2_daddr_t oldblkno;
184 panic("softdep_setup_allocdirect called");
188 softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
191 ufs2_daddr_t newblkno;
192 ufs2_daddr_t oldblkno;
198 panic("softdep_setup_allocext called");
202 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
207 ufs2_daddr_t newblkno;
208 ufs2_daddr_t oldblkno;
212 panic("softdep_setup_allocindir_page called");
216 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
221 ufs2_daddr_t newblkno;
224 panic("softdep_setup_allocindir_meta called");
228 softdep_journal_freeblocks(ip, cred, length, flags)
235 panic("softdep_journal_freeblocks called");
239 softdep_journal_fsync(ip)
243 panic("softdep_journal_fsync called");
247 softdep_setup_freeblocks(ip, length, flags)
253 panic("softdep_setup_freeblocks called");
257 softdep_freefile(pvp, ino, mode)
263 panic("softdep_freefile called");
267 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
272 struct buf *newdirbp;
276 panic("softdep_setup_directory_add called");
280 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize)
289 panic("softdep_change_directoryentry_offset called");
293 softdep_setup_remove(bp, dp, ip, isrmdir)
300 panic("softdep_setup_remove called");
304 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
312 panic("softdep_setup_directory_change called");
316 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd)
321 struct workhead *wkhd;
324 panic("%s called", __FUNCTION__);
328 softdep_setup_inofree(mp, bp, ino, wkhd)
332 struct workhead *wkhd;
335 panic("%s called", __FUNCTION__);
339 softdep_setup_unlink(dp, ip)
344 panic("%s called", __FUNCTION__);
348 softdep_setup_link(dp, ip)
353 panic("%s called", __FUNCTION__);
357 softdep_revert_link(dp, ip)
362 panic("%s called", __FUNCTION__);
366 softdep_setup_rmdir(dp, ip)
371 panic("%s called", __FUNCTION__);
375 softdep_revert_rmdir(dp, ip)
380 panic("%s called", __FUNCTION__);
384 softdep_setup_create(dp, ip)
389 panic("%s called", __FUNCTION__);
393 softdep_revert_create(dp, ip)
398 panic("%s called", __FUNCTION__);
402 softdep_setup_mkdir(dp, ip)
407 panic("%s called", __FUNCTION__);
411 softdep_revert_mkdir(dp, ip)
416 panic("%s called", __FUNCTION__);
420 softdep_setup_dotdot_link(dp, ip)
425 panic("%s called", __FUNCTION__);
429 softdep_prealloc(vp, waitok)
434 panic("%s called", __FUNCTION__);
440 softdep_journal_lookup(mp, vpp)
449 softdep_change_linkcnt(ip)
453 panic("softdep_change_linkcnt called");
457 softdep_load_inodeblock(ip)
461 panic("softdep_load_inodeblock called");
465 softdep_update_inodeblock(ip, bp, waitfor)
471 panic("softdep_update_inodeblock called");
476 struct vnode *vp; /* the "in_core" copy of the inode */
483 softdep_fsync_mountdev(vp)
491 softdep_flushworklist(oldmnt, countp, td)
492 struct mount *oldmnt;
502 softdep_sync_metadata(struct vnode *vp)
509 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor)
520 panic("softdep_slowdown called");
524 softdep_releasefile(ip)
525 struct inode *ip; /* inode with the zero effective link count */
528 panic("softdep_releasefile called");
532 softdep_request_cleanup(fs, vp, cred, resource)
543 softdep_check_suspend(struct mount *mp,
547 int secondary_writes,
548 int secondary_accwrites)
554 (void) softdep_accdeps;
556 bo = &devvp->v_bufobj;
557 ASSERT_BO_LOCKED(bo);
560 while (mp->mnt_secondary_writes != 0) {
562 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp),
563 (PUSER - 1) | PDROP, "secwr", 0);
569 * Reasons for needing more work before suspend:
570 * - Dirty buffers on devvp.
571 * - Secondary writes occurred after start of vnode sync loop
574 if (bo->bo_numoutput > 0 ||
575 bo->bo_dirty.bv_cnt > 0 ||
576 secondary_writes != 0 ||
577 mp->mnt_secondary_writes != 0 ||
578 secondary_accwrites != mp->mnt_secondary_accwrites)
585 softdep_get_depcounts(struct mount *mp,
587 int *softdepactiveaccp)
591 *softdepactiveaccp = 0;
595 softdep_buf_append(bp, wkhd)
597 struct workhead *wkhd;
600 panic("softdep_buf_appendwork called");
604 softdep_inode_append(ip, cred, wkhd)
607 struct workhead *wkhd;
610 panic("softdep_inode_appendwork called");
614 softdep_freework(wkhd)
615 struct workhead *wkhd;
618 panic("softdep_freework called");
623 FEATURE(softupdates, "FFS soft-updates support");
626 * These definitions need to be adapted to the system to which
627 * this file is being ported.
630 #define M_SOFTDEP_FLAGS (M_WAITOK)
634 #define D_BMSAFEMAP 2
636 #define D_ALLOCDIRECT 4
638 #define D_ALLOCINDIR 6
645 #define D_NEWDIRBLK 13
646 #define D_FREEWORK 14
652 #define D_JFREEBLK 20
653 #define D_JFREEFRAG 21
659 #define D_SENTINAL 27
660 #define D_LAST D_SENTINAL
662 unsigned long dep_current[D_LAST + 1];
663 unsigned long dep_total[D_LAST + 1];
664 unsigned long dep_write[D_LAST + 1];
667 static SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0,
668 "soft updates stats");
669 static SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0,
670 "total dependencies allocated");
671 static SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0,
672 "current dependencies allocated");
673 static SYSCTL_NODE(_debug_softdep, OID_AUTO, write, CTLFLAG_RW, 0,
674 "current dependencies written");
676 #define SOFTDEP_TYPE(type, str, long) \
677 static MALLOC_DEFINE(M_ ## type, #str, long); \
678 SYSCTL_ULONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \
679 &dep_total[D_ ## type], 0, ""); \
680 SYSCTL_ULONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \
681 &dep_current[D_ ## type], 0, ""); \
682 SYSCTL_ULONG(_debug_softdep_write, OID_AUTO, str, CTLFLAG_RD, \
683 &dep_write[D_ ## type], 0, "");
685 SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies");
686 SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies");
687 SOFTDEP_TYPE(BMSAFEMAP, bmsafemap,
688 "Block or frag allocated from cyl group map");
689 SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency");
690 SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode");
691 SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies");
692 SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block");
693 SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode");
694 SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode");
695 SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated");
696 SOFTDEP_TYPE(DIRADD, diradd, "New directory entry");
697 SOFTDEP_TYPE(MKDIR, mkdir, "New directory");
698 SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted");
699 SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block");
700 SOFTDEP_TYPE(FREEWORK, freework, "free an inode block");
701 SOFTDEP_TYPE(FREEDEP, freedep, "track a block free");
702 SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add");
703 SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove");
704 SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move");
705 SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block");
706 SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block");
707 SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag");
708 SOFTDEP_TYPE(JSEG, jseg, "Journal segment");
709 SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete");
710 SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency");
711 SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation");
712 SOFTDEP_TYPE(JFSYNC, jfsync, "Journal fsync complete");
714 static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes");
715 static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations");
718 * translate from workitem type to memory type
719 * MUST match the defines above, such that memtype[D_XXX] == M_XXX
721 static struct malloc_type *memtype[] = {
751 static LIST_HEAD(mkdirlist, mkdir) mkdirlisthd;
753 #define DtoM(type) (memtype[type])
756 * Names of malloc types.
758 #define TYPENAME(type) \
759 ((unsigned)(type) <= D_LAST ? memtype[type]->ks_shortdesc : "???")
761 * End system adaptation definitions.
764 #define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino)
765 #define DOT_OFFSET offsetof(struct dirtemplate, dot_ino)
768 * Forward declarations.
770 struct inodedep_hashhead;
771 struct newblk_hashhead;
772 struct pagedep_hashhead;
773 struct bmsafemap_hashhead;
776 * Private journaling structures.
779 struct jseglst jb_segs; /* TAILQ of current segments. */
780 struct jseg *jb_writeseg; /* Next write to complete. */
781 struct jseg *jb_oldestseg; /* Oldest segment with valid entries. */
782 struct jextent *jb_extent; /* Extent array. */
783 uint64_t jb_nextseq; /* Next sequence number. */
784 uint64_t jb_oldestwrseq; /* Oldest written sequence number. */
785 uint8_t jb_needseg; /* Need a forced segment. */
786 uint8_t jb_suspended; /* Did journal suspend writes? */
787 int jb_avail; /* Available extents. */
788 int jb_used; /* Last used extent. */
789 int jb_head; /* Allocator head. */
790 int jb_off; /* Allocator extent offset. */
791 int jb_blocks; /* Total disk blocks covered. */
792 int jb_free; /* Total disk blocks free. */
793 int jb_min; /* Minimum free space. */
794 int jb_low; /* Low on space. */
795 int jb_age; /* Insertion time of oldest rec. */
799 ufs2_daddr_t je_daddr; /* Disk block address. */
800 int je_blocks; /* Disk block count. */
804 * Internal function prototypes.
806 static void softdep_error(char *, int);
807 static void drain_output(struct vnode *);
808 static struct buf *getdirtybuf(struct buf *, struct mtx *, int);
809 static void clear_remove(struct thread *);
810 static void clear_inodedeps(struct thread *);
811 static void unlinked_inodedep(struct mount *, struct inodedep *);
812 static void clear_unlinked_inodedep(struct inodedep *);
813 static struct inodedep *first_unlinked_inodedep(struct ufsmount *);
814 static int flush_pagedep_deps(struct vnode *, struct mount *,
816 static int free_pagedep(struct pagedep *);
817 static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t);
818 static int flush_inodedep_deps(struct vnode *, struct mount *, ino_t);
819 static int flush_deplist(struct allocdirectlst *, int, int *);
820 static int sync_cgs(struct mount *, int);
821 static int handle_written_filepage(struct pagedep *, struct buf *);
822 static int handle_written_sbdep(struct sbdep *, struct buf *);
823 static void initiate_write_sbdep(struct sbdep *);
824 static void diradd_inode_written(struct diradd *, struct inodedep *);
825 static int handle_written_indirdep(struct indirdep *, struct buf *,
827 static int handle_written_inodeblock(struct inodedep *, struct buf *);
828 static int jnewblk_rollforward(struct jnewblk *, struct fs *, struct cg *,
830 static int handle_written_bmsafemap(struct bmsafemap *, struct buf *);
831 static void handle_written_jaddref(struct jaddref *);
832 static void handle_written_jremref(struct jremref *);
833 static void handle_written_jseg(struct jseg *, struct buf *);
834 static void handle_written_jnewblk(struct jnewblk *);
835 static void handle_written_jblkdep(struct jblkdep *);
836 static void handle_written_jfreefrag(struct jfreefrag *);
837 static void complete_jseg(struct jseg *);
838 static void complete_jsegs(struct jseg *);
839 static void jseg_write(struct ufsmount *ump, struct jseg *, uint8_t *);
840 static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *);
841 static void jremref_write(struct jremref *, struct jseg *, uint8_t *);
842 static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *);
843 static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *);
844 static void jfsync_write(struct jfsync *, struct jseg *, uint8_t *data);
845 static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *);
846 static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *);
847 static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *);
848 static inline void inoref_write(struct inoref *, struct jseg *,
850 static void handle_allocdirect_partdone(struct allocdirect *,
852 static struct jnewblk *cancel_newblk(struct newblk *, struct worklist *,
854 static void indirdep_complete(struct indirdep *);
855 static int indirblk_lookup(struct mount *, ufs2_daddr_t);
856 static void indirblk_insert(struct freework *);
857 static void indirblk_remove(struct freework *);
858 static void handle_allocindir_partdone(struct allocindir *);
859 static void initiate_write_filepage(struct pagedep *, struct buf *);
860 static void initiate_write_indirdep(struct indirdep*, struct buf *);
861 static void handle_written_mkdir(struct mkdir *, int);
862 static int jnewblk_rollback(struct jnewblk *, struct fs *, struct cg *,
864 static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *);
865 static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *);
866 static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *);
867 static void handle_workitem_freefile(struct freefile *);
868 static int handle_workitem_remove(struct dirrem *, int);
869 static struct dirrem *newdirrem(struct buf *, struct inode *,
870 struct inode *, int, struct dirrem **);
871 static struct indirdep *indirdep_lookup(struct mount *, struct inode *,
873 static void cancel_indirdep(struct indirdep *, struct buf *,
875 static void free_indirdep(struct indirdep *);
876 static void free_diradd(struct diradd *, struct workhead *);
877 static void merge_diradd(struct inodedep *, struct diradd *);
878 static void complete_diradd(struct diradd *);
879 static struct diradd *diradd_lookup(struct pagedep *, int);
880 static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *,
882 static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *,
884 static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *,
885 struct jremref *, struct jremref *);
886 static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *,
888 static void cancel_allocindir(struct allocindir *, struct buf *bp,
889 struct freeblks *, int);
890 static int setup_trunc_indir(struct freeblks *, struct inode *,
891 ufs_lbn_t, ufs_lbn_t, ufs2_daddr_t);
892 static void complete_trunc_indir(struct freework *);
893 static void trunc_indirdep(struct indirdep *, struct freeblks *, struct buf *,
895 static void complete_mkdir(struct mkdir *);
896 static void free_newdirblk(struct newdirblk *);
897 static void free_jremref(struct jremref *);
898 static void free_jaddref(struct jaddref *);
899 static void free_jsegdep(struct jsegdep *);
900 static void free_jsegs(struct jblocks *);
901 static void rele_jseg(struct jseg *);
902 static void free_jseg(struct jseg *, struct jblocks *);
903 static void free_jnewblk(struct jnewblk *);
904 static void free_jblkdep(struct jblkdep *);
905 static void free_jfreefrag(struct jfreefrag *);
906 static void free_freedep(struct freedep *);
907 static void journal_jremref(struct dirrem *, struct jremref *,
909 static void cancel_jnewblk(struct jnewblk *, struct workhead *);
910 static int cancel_jaddref(struct jaddref *, struct inodedep *,
912 static void cancel_jfreefrag(struct jfreefrag *);
913 static inline void setup_freedirect(struct freeblks *, struct inode *,
915 static inline void setup_freeext(struct freeblks *, struct inode *, int, int);
916 static inline void setup_freeindir(struct freeblks *, struct inode *, int,
918 static inline struct freeblks *newfreeblks(struct mount *, struct inode *);
919 static void freeblks_free(struct ufsmount *, struct freeblks *, int);
920 static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t);
921 ufs2_daddr_t blkcount(struct fs *, ufs2_daddr_t, off_t);
922 static int trunc_check_buf(struct buf *, int *, ufs_lbn_t, int, int);
923 static void trunc_dependencies(struct inode *, struct freeblks *, ufs_lbn_t,
925 static void trunc_pages(struct inode *, off_t, ufs2_daddr_t, int);
926 static int cancel_pagedep(struct pagedep *, struct freeblks *, int);
927 static int deallocate_dependencies(struct buf *, struct freeblks *, int);
928 static void newblk_freefrag(struct newblk*);
929 static void free_newblk(struct newblk *);
930 static void cancel_allocdirect(struct allocdirectlst *,
931 struct allocdirect *, struct freeblks *);
932 static int check_inode_unwritten(struct inodedep *);
933 static int free_inodedep(struct inodedep *);
934 static void freework_freeblock(struct freework *);
935 static void freework_enqueue(struct freework *);
936 static int handle_workitem_freeblocks(struct freeblks *, int);
937 static int handle_complete_freeblocks(struct freeblks *, int);
938 static void handle_workitem_indirblk(struct freework *);
939 static void handle_written_freework(struct freework *);
940 static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *);
941 static struct worklist *jnewblk_merge(struct worklist *, struct worklist *,
943 static struct freefrag *setup_allocindir_phase2(struct buf *, struct inode *,
944 struct inodedep *, struct allocindir *, ufs_lbn_t);
945 static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t,
946 ufs2_daddr_t, ufs_lbn_t);
947 static void handle_workitem_freefrag(struct freefrag *);
948 static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long,
950 static void allocdirect_merge(struct allocdirectlst *,
951 struct allocdirect *, struct allocdirect *);
952 static struct freefrag *allocindir_merge(struct allocindir *,
953 struct allocindir *);
954 static int bmsafemap_find(struct bmsafemap_hashhead *, struct mount *, int,
955 struct bmsafemap **);
956 static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *,
957 int cg, struct bmsafemap *);
958 static int newblk_find(struct newblk_hashhead *, struct mount *, ufs2_daddr_t,
959 int, struct newblk **);
960 static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **);
961 static int inodedep_find(struct inodedep_hashhead *, struct fs *, ino_t,
963 static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **);
964 static int pagedep_lookup(struct mount *, struct buf *bp, ino_t, ufs_lbn_t,
965 int, struct pagedep **);
966 static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t,
967 struct mount *mp, int, struct pagedep **);
968 static void pause_timer(void *);
969 static int request_cleanup(struct mount *, int);
970 static int process_worklist_item(struct mount *, int, int);
971 static void process_removes(struct vnode *);
972 static void process_truncates(struct vnode *);
973 static void jwork_move(struct workhead *, struct workhead *);
974 static void jwork_insert(struct workhead *, struct jsegdep *);
975 static void add_to_worklist(struct worklist *, int);
976 static void wake_worklist(struct worklist *);
977 static void wait_worklist(struct worklist *, char *);
978 static void remove_from_worklist(struct worklist *);
979 static void softdep_flush(void);
980 static void softdep_flushjournal(struct mount *);
981 static int softdep_speedup(void);
982 static void worklist_speedup(void);
983 static int journal_mount(struct mount *, struct fs *, struct ucred *);
984 static void journal_unmount(struct mount *);
985 static int journal_space(struct ufsmount *, int);
986 static void journal_suspend(struct ufsmount *);
987 static int journal_unsuspend(struct ufsmount *ump);
988 static void softdep_prelink(struct vnode *, struct vnode *);
989 static void add_to_journal(struct worklist *);
990 static void remove_from_journal(struct worklist *);
991 static void softdep_process_journal(struct mount *, struct worklist *, int);
992 static struct jremref *newjremref(struct dirrem *, struct inode *,
993 struct inode *ip, off_t, nlink_t);
994 static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t,
996 static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t,
998 static inline struct jsegdep *inoref_jseg(struct inoref *);
999 static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t);
1000 static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t,
1002 static struct jtrunc *newjtrunc(struct freeblks *, off_t, int);
1003 static void move_newblock_dep(struct jaddref *, struct inodedep *);
1004 static void cancel_jfreeblk(struct freeblks *, ufs2_daddr_t);
1005 static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *,
1006 ufs2_daddr_t, long, ufs_lbn_t);
1007 static struct freework *newfreework(struct ufsmount *, struct freeblks *,
1008 struct freework *, ufs_lbn_t, ufs2_daddr_t, int, int, int);
1009 static int jwait(struct worklist *, int);
1010 static struct inodedep *inodedep_lookup_ip(struct inode *);
1011 static int bmsafemap_backgroundwrite(struct bmsafemap *, struct buf *);
1012 static struct freefile *handle_bufwait(struct inodedep *, struct workhead *);
1013 static void handle_jwork(struct workhead *);
1014 static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *,
1016 static struct jblocks *jblocks_create(void);
1017 static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *);
1018 static void jblocks_free(struct jblocks *, struct mount *, int);
1019 static void jblocks_destroy(struct jblocks *);
1020 static void jblocks_add(struct jblocks *, ufs2_daddr_t, int);
1023 * Exported softdep operations.
1025 static void softdep_disk_io_initiation(struct buf *);
1026 static void softdep_disk_write_complete(struct buf *);
1027 static void softdep_deallocate_dependencies(struct buf *);
1028 static int softdep_count_dependencies(struct buf *bp, int);
1030 static struct mtx lk;
1031 MTX_SYSINIT(softdep_lock, &lk, "Softdep Lock", MTX_DEF);
1033 #define TRY_ACQUIRE_LOCK(lk) mtx_trylock(lk)
1034 #define ACQUIRE_LOCK(lk) mtx_lock(lk)
1035 #define FREE_LOCK(lk) mtx_unlock(lk)
1037 #define BUF_AREC(bp) lockallowrecurse(&(bp)->b_lock)
1038 #define BUF_NOREC(bp) lockdisablerecurse(&(bp)->b_lock)
1041 * Worklist queue management.
1042 * These routines require that the lock be held.
1044 #ifndef /* NOT */ DEBUG
1045 #define WORKLIST_INSERT(head, item) do { \
1046 (item)->wk_state |= ONWORKLIST; \
1047 LIST_INSERT_HEAD(head, item, wk_list); \
1049 #define WORKLIST_REMOVE(item) do { \
1050 (item)->wk_state &= ~ONWORKLIST; \
1051 LIST_REMOVE(item, wk_list); \
1053 #define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT
1054 #define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE
1057 static void worklist_insert(struct workhead *, struct worklist *, int);
1058 static void worklist_remove(struct worklist *, int);
1060 #define WORKLIST_INSERT(head, item) worklist_insert(head, item, 1)
1061 #define WORKLIST_INSERT_UNLOCKED(head, item) worklist_insert(head, item, 0)
1062 #define WORKLIST_REMOVE(item) worklist_remove(item, 1)
1063 #define WORKLIST_REMOVE_UNLOCKED(item) worklist_remove(item, 0)
1066 worklist_insert(head, item, locked)
1067 struct workhead *head;
1068 struct worklist *item;
1073 mtx_assert(&lk, MA_OWNED);
1074 if (item->wk_state & ONWORKLIST)
1075 panic("worklist_insert: %p %s(0x%X) already on list",
1076 item, TYPENAME(item->wk_type), item->wk_state);
1077 item->wk_state |= ONWORKLIST;
1078 LIST_INSERT_HEAD(head, item, wk_list);
1082 worklist_remove(item, locked)
1083 struct worklist *item;
1088 mtx_assert(&lk, MA_OWNED);
1089 if ((item->wk_state & ONWORKLIST) == 0)
1090 panic("worklist_remove: %p %s(0x%X) not on list",
1091 item, TYPENAME(item->wk_type), item->wk_state);
1092 item->wk_state &= ~ONWORKLIST;
1093 LIST_REMOVE(item, wk_list);
1098 * Merge two jsegdeps keeping only the oldest one as newer references
1099 * can't be discarded until after older references.
1101 static inline struct jsegdep *
1102 jsegdep_merge(struct jsegdep *one, struct jsegdep *two)
1104 struct jsegdep *swp;
1109 if (one->jd_seg->js_seq > two->jd_seg->js_seq) {
1114 WORKLIST_REMOVE(&two->jd_list);
1121 * If two freedeps are compatible free one to reduce list size.
1123 static inline struct freedep *
1124 freedep_merge(struct freedep *one, struct freedep *two)
1129 if (one->fd_freework == two->fd_freework) {
1130 WORKLIST_REMOVE(&two->fd_list);
1137 * Move journal work from one list to another. Duplicate freedeps and
1138 * jsegdeps are coalesced to keep the lists as small as possible.
1141 jwork_move(dst, src)
1142 struct workhead *dst;
1143 struct workhead *src;
1145 struct freedep *freedep;
1146 struct jsegdep *jsegdep;
1147 struct worklist *wkn;
1148 struct worklist *wk;
1151 ("jwork_move: dst == src"));
1154 LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) {
1155 if (wk->wk_type == D_JSEGDEP)
1156 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep);
1157 if (wk->wk_type == D_FREEDEP)
1158 freedep = freedep_merge(WK_FREEDEP(wk), freedep);
1161 mtx_assert(&lk, MA_OWNED);
1162 while ((wk = LIST_FIRST(src)) != NULL) {
1163 WORKLIST_REMOVE(wk);
1164 WORKLIST_INSERT(dst, wk);
1165 if (wk->wk_type == D_JSEGDEP) {
1166 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep);
1169 if (wk->wk_type == D_FREEDEP)
1170 freedep = freedep_merge(WK_FREEDEP(wk), freedep);
1175 jwork_insert(dst, jsegdep)
1176 struct workhead *dst;
1177 struct jsegdep *jsegdep;
1179 struct jsegdep *jsegdepn;
1180 struct worklist *wk;
1182 LIST_FOREACH(wk, dst, wk_list)
1183 if (wk->wk_type == D_JSEGDEP)
1186 WORKLIST_INSERT(dst, &jsegdep->jd_list);
1189 jsegdepn = WK_JSEGDEP(wk);
1190 if (jsegdep->jd_seg->js_seq < jsegdepn->jd_seg->js_seq) {
1191 WORKLIST_REMOVE(wk);
1192 free_jsegdep(jsegdepn);
1193 WORKLIST_INSERT(dst, &jsegdep->jd_list);
1195 free_jsegdep(jsegdep);
1199 * Routines for tracking and managing workitems.
1201 static void workitem_free(struct worklist *, int);
1202 static void workitem_alloc(struct worklist *, int, struct mount *);
1204 #define WORKITEM_FREE(item, type) workitem_free((struct worklist *)(item), (type))
1207 workitem_free(item, type)
1208 struct worklist *item;
1211 struct ufsmount *ump;
1212 mtx_assert(&lk, MA_OWNED);
1215 if (item->wk_state & ONWORKLIST)
1216 panic("workitem_free: %s(0x%X) still on list",
1217 TYPENAME(item->wk_type), item->wk_state);
1218 if (item->wk_type != type)
1219 panic("workitem_free: type mismatch %s != %s",
1220 TYPENAME(item->wk_type), TYPENAME(type));
1222 if (item->wk_state & IOWAITING)
1224 ump = VFSTOUFS(item->wk_mp);
1225 if (--ump->softdep_deps == 0 && ump->softdep_req)
1226 wakeup(&ump->softdep_deps);
1227 dep_current[type]--;
1228 free(item, DtoM(type));
1232 workitem_alloc(item, type, mp)
1233 struct worklist *item;
1237 struct ufsmount *ump;
1239 item->wk_type = type;
1245 dep_current[type]++;
1247 ump->softdep_deps++;
1248 ump->softdep_accdeps++;
1253 * Workitem queue management
1255 static int max_softdeps; /* maximum number of structs before slowdown */
1256 static int maxindirdeps = 50; /* max number of indirdeps before slowdown */
1257 static int tickdelay = 2; /* number of ticks to pause during slowdown */
1258 static int proc_waiting; /* tracks whether we have a timeout posted */
1259 static int *stat_countp; /* statistic to count in proc_waiting timeout */
1260 static struct callout softdep_callout;
1261 static int req_pending;
1262 static int req_clear_inodedeps; /* syncer process flush some inodedeps */
1263 static int req_clear_remove; /* syncer process flush some freeblks */
1264 static int softdep_flushcache = 0; /* Should we do BIO_FLUSH? */
1267 * runtime statistics
1269 static int stat_worklist_push; /* number of worklist cleanups */
1270 static int stat_blk_limit_push; /* number of times block limit neared */
1271 static int stat_ino_limit_push; /* number of times inode limit neared */
1272 static int stat_blk_limit_hit; /* number of times block slowdown imposed */
1273 static int stat_ino_limit_hit; /* number of times inode slowdown imposed */
1274 static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */
1275 static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */
1276 static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */
1277 static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */
1278 static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */
1279 static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */
1280 static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */
1281 static int stat_journal_min; /* Times hit journal min threshold */
1282 static int stat_journal_low; /* Times hit journal low threshold */
1283 static int stat_journal_wait; /* Times blocked in jwait(). */
1284 static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */
1285 static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */
1286 static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */
1287 static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */
1288 static int stat_cleanup_high_delay; /* Maximum cleanup delay (in ticks) */
1289 static int stat_cleanup_blkrequests; /* Number of block cleanup requests */
1290 static int stat_cleanup_inorequests; /* Number of inode cleanup requests */
1291 static int stat_cleanup_retries; /* Number of cleanups that needed to flush */
1292 static int stat_cleanup_failures; /* Number of cleanup requests that failed */
1294 SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW,
1295 &max_softdeps, 0, "");
1296 SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW,
1298 SYSCTL_INT(_debug_softdep, OID_AUTO, maxindirdeps, CTLFLAG_RW,
1299 &maxindirdeps, 0, "");
1300 SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push, CTLFLAG_RW,
1301 &stat_worklist_push, 0,"");
1302 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push, CTLFLAG_RW,
1303 &stat_blk_limit_push, 0,"");
1304 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push, CTLFLAG_RW,
1305 &stat_ino_limit_push, 0,"");
1306 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit, CTLFLAG_RW,
1307 &stat_blk_limit_hit, 0, "");
1308 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit, CTLFLAG_RW,
1309 &stat_ino_limit_hit, 0, "");
1310 SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit, CTLFLAG_RW,
1311 &stat_sync_limit_hit, 0, "");
1312 SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs, CTLFLAG_RW,
1313 &stat_indir_blk_ptrs, 0, "");
1314 SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap, CTLFLAG_RW,
1315 &stat_inode_bitmap, 0, "");
1316 SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs, CTLFLAG_RW,
1317 &stat_direct_blk_ptrs, 0, "");
1318 SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry, CTLFLAG_RW,
1319 &stat_dir_entry, 0, "");
1320 SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback, CTLFLAG_RW,
1321 &stat_jaddref, 0, "");
1322 SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback, CTLFLAG_RW,
1323 &stat_jnewblk, 0, "");
1324 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low, CTLFLAG_RW,
1325 &stat_journal_low, 0, "");
1326 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min, CTLFLAG_RW,
1327 &stat_journal_min, 0, "");
1328 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait, CTLFLAG_RW,
1329 &stat_journal_wait, 0, "");
1330 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage, CTLFLAG_RW,
1331 &stat_jwait_filepage, 0, "");
1332 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks, CTLFLAG_RW,
1333 &stat_jwait_freeblks, 0, "");
1334 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode, CTLFLAG_RW,
1335 &stat_jwait_inode, 0, "");
1336 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk, CTLFLAG_RW,
1337 &stat_jwait_newblk, 0, "");
1338 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_blkrequests, CTLFLAG_RW,
1339 &stat_cleanup_blkrequests, 0, "");
1340 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_inorequests, CTLFLAG_RW,
1341 &stat_cleanup_inorequests, 0, "");
1342 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_high_delay, CTLFLAG_RW,
1343 &stat_cleanup_high_delay, 0, "");
1344 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_retries, CTLFLAG_RW,
1345 &stat_cleanup_retries, 0, "");
1346 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_failures, CTLFLAG_RW,
1347 &stat_cleanup_failures, 0, "");
1348 SYSCTL_INT(_debug_softdep, OID_AUTO, flushcache, CTLFLAG_RW,
1349 &softdep_flushcache, 0, "");
1351 SYSCTL_DECL(_vfs_ffs);
1353 LIST_HEAD(bmsafemap_hashhead, bmsafemap) *bmsafemap_hashtbl;
1354 static u_long bmsafemap_hash; /* size of hash table - 1 */
1356 static int compute_summary_at_mount = 0; /* Whether to recompute the summary at mount time */
1357 SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW,
1358 &compute_summary_at_mount, 0, "Recompute summary at mount");
1360 static struct proc *softdepproc;
1361 static struct kproc_desc softdep_kp = {
1366 SYSINIT(sdproc, SI_SUB_KTHREAD_UPDATE, SI_ORDER_ANY, kproc_start,
1374 struct ufsmount *ump;
1381 td->td_pflags |= TDP_NORUNNINGBUF;
1384 kproc_suspend_check(softdepproc);
1385 vfslocked = VFS_LOCK_GIANT((struct mount *)NULL);
1388 * If requested, try removing inode or removal dependencies.
1390 if (req_clear_inodedeps) {
1391 clear_inodedeps(td);
1392 req_clear_inodedeps -= 1;
1393 wakeup_one(&proc_waiting);
1395 if (req_clear_remove) {
1397 req_clear_remove -= 1;
1398 wakeup_one(&proc_waiting);
1401 VFS_UNLOCK_GIANT(vfslocked);
1402 remaining = progress = 0;
1403 mtx_lock(&mountlist_mtx);
1404 for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
1405 nmp = TAILQ_NEXT(mp, mnt_list);
1406 if (MOUNTEDSOFTDEP(mp) == 0)
1408 if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
1410 vfslocked = VFS_LOCK_GIANT(mp);
1411 progress += softdep_process_worklist(mp, 0);
1413 remaining += ump->softdep_on_worklist;
1414 VFS_UNLOCK_GIANT(vfslocked);
1415 mtx_lock(&mountlist_mtx);
1416 nmp = TAILQ_NEXT(mp, mnt_list);
1419 mtx_unlock(&mountlist_mtx);
1420 if (remaining && progress)
1424 msleep(&req_pending, &lk, PVM, "sdflush", hz);
1431 worklist_speedup(void)
1433 mtx_assert(&lk, MA_OWNED);
1434 if (req_pending == 0) {
1436 wakeup(&req_pending);
1441 softdep_speedup(void)
1446 return speedup_syncer();
1450 * Add an item to the end of the work queue.
1451 * This routine requires that the lock be held.
1452 * This is the only routine that adds items to the list.
1453 * The following routine is the only one that removes items
1454 * and does so in order from first to last.
1457 #define WK_HEAD 0x0001 /* Add to HEAD. */
1458 #define WK_NODELAY 0x0002 /* Process immediately. */
1461 add_to_worklist(wk, flags)
1462 struct worklist *wk;
1465 struct ufsmount *ump;
1467 mtx_assert(&lk, MA_OWNED);
1468 ump = VFSTOUFS(wk->wk_mp);
1469 if (wk->wk_state & ONWORKLIST)
1470 panic("add_to_worklist: %s(0x%X) already on list",
1471 TYPENAME(wk->wk_type), wk->wk_state);
1472 wk->wk_state |= ONWORKLIST;
1473 if (ump->softdep_on_worklist == 0) {
1474 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list);
1475 ump->softdep_worklist_tail = wk;
1476 } else if (flags & WK_HEAD) {
1477 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list);
1479 LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list);
1480 ump->softdep_worklist_tail = wk;
1482 ump->softdep_on_worklist += 1;
1483 if (flags & WK_NODELAY)
1488 * Remove the item to be processed. If we are removing the last
1489 * item on the list, we need to recalculate the tail pointer.
1492 remove_from_worklist(wk)
1493 struct worklist *wk;
1495 struct ufsmount *ump;
1497 ump = VFSTOUFS(wk->wk_mp);
1498 WORKLIST_REMOVE(wk);
1499 if (ump->softdep_worklist_tail == wk)
1500 ump->softdep_worklist_tail =
1501 (struct worklist *)wk->wk_list.le_prev;
1502 ump->softdep_on_worklist -= 1;
1507 struct worklist *wk;
1509 if (wk->wk_state & IOWAITING) {
1510 wk->wk_state &= ~IOWAITING;
1516 wait_worklist(wk, wmesg)
1517 struct worklist *wk;
1521 wk->wk_state |= IOWAITING;
1522 msleep(wk, &lk, PVM, wmesg, 0);
1526 * Process that runs once per second to handle items in the background queue.
1528 * Note that we ensure that everything is done in the order in which they
1529 * appear in the queue. The code below depends on this property to ensure
1530 * that blocks of a file are freed before the inode itself is freed. This
1531 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated
1532 * until all the old ones have been purged from the dependency lists.
1535 softdep_process_worklist(mp, full)
1539 struct thread *td = curthread;
1541 struct ufsmount *ump;
1544 KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp"));
1546 * Record the process identifier of our caller so that we can give
1547 * this process preferential treatment in request_cleanup below.
1552 starttime = time_second;
1553 softdep_process_journal(mp, NULL, full?MNT_WAIT:0);
1554 while (ump->softdep_on_worklist > 0) {
1555 if ((cnt = process_worklist_item(mp, 10, LK_NOWAIT)) == 0)
1560 * If requested, try removing inode or removal dependencies.
1562 if (req_clear_inodedeps) {
1563 clear_inodedeps(td);
1564 req_clear_inodedeps -= 1;
1565 wakeup_one(&proc_waiting);
1567 if (req_clear_remove) {
1569 req_clear_remove -= 1;
1570 wakeup_one(&proc_waiting);
1573 * We do not generally want to stop for buffer space, but if
1574 * we are really being a buffer hog, we will stop and wait.
1576 if (should_yield()) {
1578 kern_yield(PRI_UNCHANGED);
1583 * Never allow processing to run for more than one
1584 * second. Otherwise the other mountpoints may get
1585 * excessively backlogged.
1587 if (!full && starttime != time_second)
1591 journal_unsuspend(ump);
1597 * Process all removes associated with a vnode if we are running out of
1598 * journal space. Any other process which attempts to flush these will
1599 * be unable as we have the vnodes locked.
1605 struct inodedep *inodedep;
1606 struct dirrem *dirrem;
1610 mtx_assert(&lk, MA_OWNED);
1613 inum = VTOI(vp)->i_number;
1616 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0)
1618 LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) {
1620 * If another thread is trying to lock this vnode
1621 * it will fail but we must wait for it to do so
1622 * before we can proceed.
1624 if (dirrem->dm_state & INPROGRESS) {
1625 wait_worklist(&dirrem->dm_list, "pwrwait");
1628 if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) ==
1629 (COMPLETE | ONWORKLIST))
1634 remove_from_worklist(&dirrem->dm_list);
1636 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT))
1637 panic("process_removes: suspended filesystem");
1638 handle_workitem_remove(dirrem, 0);
1639 vn_finished_secondary_write(mp);
1645 * Process all truncations associated with a vnode if we are running out
1646 * of journal space. This is called when the vnode lock is already held
1647 * and no other process can clear the truncation. This function returns
1648 * a value greater than zero if it did any work.
1651 process_truncates(vp)
1654 struct inodedep *inodedep;
1655 struct freeblks *freeblks;
1660 mtx_assert(&lk, MA_OWNED);
1663 inum = VTOI(vp)->i_number;
1665 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0)
1668 TAILQ_FOREACH(freeblks, &inodedep->id_freeblklst, fb_next) {
1669 /* Journal entries not yet written. */
1670 if (!LIST_EMPTY(&freeblks->fb_jblkdephd)) {
1672 &freeblks->fb_jblkdephd)->jb_list,
1676 /* Another thread is executing this item. */
1677 if (freeblks->fb_state & INPROGRESS) {
1678 wait_worklist(&freeblks->fb_list, "ptrwait");
1681 /* Freeblks is waiting on a inode write. */
1682 if ((freeblks->fb_state & COMPLETE) == 0) {
1688 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST)) ==
1689 (ALLCOMPLETE | ONWORKLIST)) {
1690 remove_from_worklist(&freeblks->fb_list);
1691 freeblks->fb_state |= INPROGRESS;
1693 if (vn_start_secondary_write(NULL, &mp,
1695 panic("process_truncates: "
1696 "suspended filesystem");
1697 handle_workitem_freeblocks(freeblks, 0);
1698 vn_finished_secondary_write(mp);
1702 if (freeblks->fb_cgwait)
1707 sync_cgs(mp, MNT_WAIT);
1708 ffs_sync_snap(mp, MNT_WAIT);
1712 if (freeblks == NULL)
1719 * Process one item on the worklist.
1722 process_worklist_item(mp, target, flags)
1727 struct worklist sintenel;
1728 struct worklist *wk;
1729 struct ufsmount *ump;
1733 mtx_assert(&lk, MA_OWNED);
1734 KASSERT(mp != NULL, ("process_worklist_item: NULL mp"));
1736 * If we are being called because of a process doing a
1737 * copy-on-write, then it is not safe to write as we may
1738 * recurse into the copy-on-write routine.
1740 if (curthread->td_pflags & TDP_COWINPROGRESS)
1742 PHOLD(curproc); /* Don't let the stack go away. */
1745 sintenel.wk_mp = NULL;
1746 sintenel.wk_type = D_SENTINAL;
1747 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, &sintenel, wk_list);
1748 for (wk = LIST_NEXT(&sintenel, wk_list); wk != NULL;
1749 wk = LIST_NEXT(&sintenel, wk_list)) {
1750 if (wk->wk_type == D_SENTINAL) {
1751 LIST_REMOVE(&sintenel, wk_list);
1752 LIST_INSERT_AFTER(wk, &sintenel, wk_list);
1755 if (wk->wk_state & INPROGRESS)
1756 panic("process_worklist_item: %p already in progress.",
1758 wk->wk_state |= INPROGRESS;
1759 remove_from_worklist(wk);
1761 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT))
1762 panic("process_worklist_item: suspended filesystem");
1763 switch (wk->wk_type) {
1765 /* removal of a directory entry */
1766 error = handle_workitem_remove(WK_DIRREM(wk), flags);
1770 /* releasing blocks and/or fragments from a file */
1771 error = handle_workitem_freeblocks(WK_FREEBLKS(wk),
1776 /* releasing a fragment when replaced as a file grows */
1777 handle_workitem_freefrag(WK_FREEFRAG(wk));
1782 /* releasing an inode when its link count drops to 0 */
1783 handle_workitem_freefile(WK_FREEFILE(wk));
1788 panic("%s_process_worklist: Unknown type %s",
1789 "softdep", TYPENAME(wk->wk_type));
1792 vn_finished_secondary_write(mp);
1795 if (++matchcnt == target)
1800 * We have to retry the worklist item later. Wake up any
1801 * waiters who may be able to complete it immediately and
1802 * add the item back to the head so we don't try to execute
1805 wk->wk_state &= ~INPROGRESS;
1807 add_to_worklist(wk, WK_HEAD);
1809 LIST_REMOVE(&sintenel, wk_list);
1810 /* Sentinal could've become the tail from remove_from_worklist. */
1811 if (ump->softdep_worklist_tail == &sintenel)
1812 ump->softdep_worklist_tail =
1813 (struct worklist *)sintenel.wk_list.le_prev;
1819 * Move dependencies from one buffer to another.
1822 softdep_move_dependencies(oldbp, newbp)
1826 struct worklist *wk, *wktail;
1832 while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) {
1833 LIST_REMOVE(wk, wk_list);
1834 if (wk->wk_type == D_BMSAFEMAP &&
1835 bmsafemap_backgroundwrite(WK_BMSAFEMAP(wk), newbp))
1838 LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list);
1840 LIST_INSERT_AFTER(wktail, wk, wk_list);
1849 * Purge the work list of all items associated with a particular mount point.
1852 softdep_flushworklist(oldmnt, countp, td)
1853 struct mount *oldmnt;
1857 struct vnode *devvp;
1858 int count, error = 0;
1859 struct ufsmount *ump;
1862 * Alternately flush the block device associated with the mount
1863 * point and process any dependencies that the flushing
1864 * creates. We continue until no more worklist dependencies
1868 ump = VFSTOUFS(oldmnt);
1869 devvp = ump->um_devvp;
1870 while ((count = softdep_process_worklist(oldmnt, 1)) > 0) {
1872 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1873 error = VOP_FSYNC(devvp, MNT_WAIT, td);
1874 VOP_UNLOCK(devvp, 0);
1882 softdep_waitidle(struct mount *mp)
1884 struct ufsmount *ump;
1890 for (i = 0; i < 10 && ump->softdep_deps; i++) {
1891 ump->softdep_req = 1;
1892 if (ump->softdep_on_worklist)
1893 panic("softdep_waitidle: work added after flush.");
1894 msleep(&ump->softdep_deps, &lk, PVM, "softdeps", 1);
1896 ump->softdep_req = 0;
1901 printf("softdep_waitidle: Failed to flush worklist for %p\n",
1909 * Flush all vnodes and worklist items associated with a specified mount point.
1912 softdep_flushfiles(oldmnt, flags, td)
1913 struct mount *oldmnt;
1917 int error, depcount, loopcnt, retry_flush_count, retry;
1920 retry_flush_count = 3;
1925 * Alternately flush the vnodes associated with the mount
1926 * point and process any dependencies that the flushing
1927 * creates. In theory, this loop can happen at most twice,
1928 * but we give it a few extra just to be sure.
1930 for (; loopcnt > 0; loopcnt--) {
1932 * Do another flush in case any vnodes were brought in
1933 * as part of the cleanup operations.
1935 if ((error = ffs_flushfiles(oldmnt, flags, td)) != 0)
1937 if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 ||
1942 * If we are unmounting then it is an error to fail. If we
1943 * are simply trying to downgrade to read-only, then filesystem
1944 * activity can keep us busy forever, so we just fail with EBUSY.
1947 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT)
1948 panic("softdep_flushfiles: looping");
1952 error = softdep_waitidle(oldmnt);
1954 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) {
1957 KASSERT((oldmnt->mnt_kern_flag & MNTK_NOINSMNTQ) != 0,
1958 ("softdep_flushfiles: !MNTK_NOINSMNTQ"));
1959 if (oldmnt->mnt_nvnodelistsize > 0) {
1960 if (--retry_flush_count > 0) {
1966 MNT_IUNLOCK(oldmnt);
1975 * Structure hashing.
1977 * There are three types of structures that can be looked up:
1978 * 1) pagedep structures identified by mount point, inode number,
1979 * and logical block.
1980 * 2) inodedep structures identified by mount point and inode number.
1981 * 3) newblk structures identified by mount point and
1982 * physical block number.
1984 * The "pagedep" and "inodedep" dependency structures are hashed
1985 * separately from the file blocks and inodes to which they correspond.
1986 * This separation helps when the in-memory copy of an inode or
1987 * file block must be replaced. It also obviates the need to access
1988 * an inode or file page when simply updating (or de-allocating)
1989 * dependency structures. Lookup of newblk structures is needed to
1990 * find newly allocated blocks when trying to associate them with
1991 * their allocdirect or allocindir structure.
1993 * The lookup routines optionally create and hash a new instance when
1994 * an existing entry is not found.
1996 #define DEPALLOC 0x0001 /* allocate structure if lookup fails */
1997 #define NODELAY 0x0002 /* cannot do background work */
2000 * Structures and routines associated with pagedep caching.
2002 LIST_HEAD(pagedep_hashhead, pagedep) *pagedep_hashtbl;
2003 u_long pagedep_hash; /* size of hash table - 1 */
2004 #define PAGEDEP_HASH(mp, inum, lbn) \
2005 (&pagedep_hashtbl[((((register_t)(mp)) >> 13) + (inum) + (lbn)) & \
2009 pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp)
2010 struct pagedep_hashhead *pagedephd;
2015 struct pagedep **pagedeppp;
2017 struct pagedep *pagedep;
2019 LIST_FOREACH(pagedep, pagedephd, pd_hash) {
2020 if (ino == pagedep->pd_ino && lbn == pagedep->pd_lbn &&
2021 mp == pagedep->pd_list.wk_mp) {
2022 *pagedeppp = pagedep;
2030 * Look up a pagedep. Return 1 if found, 0 otherwise.
2031 * If not found, allocate if DEPALLOC flag is passed.
2032 * Found or allocated entry is returned in pagedeppp.
2033 * This routine must be called with splbio interrupts blocked.
2036 pagedep_lookup(mp, bp, ino, lbn, flags, pagedeppp)
2042 struct pagedep **pagedeppp;
2044 struct pagedep *pagedep;
2045 struct pagedep_hashhead *pagedephd;
2046 struct worklist *wk;
2050 mtx_assert(&lk, MA_OWNED);
2052 LIST_FOREACH(wk, &bp->b_dep, wk_list) {
2053 if (wk->wk_type == D_PAGEDEP) {
2054 *pagedeppp = WK_PAGEDEP(wk);
2059 pagedephd = PAGEDEP_HASH(mp, ino, lbn);
2060 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp);
2062 if (((*pagedeppp)->pd_state & ONWORKLIST) == 0 && bp)
2063 WORKLIST_INSERT(&bp->b_dep, &(*pagedeppp)->pd_list);
2066 if ((flags & DEPALLOC) == 0)
2069 pagedep = malloc(sizeof(struct pagedep),
2070 M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO);
2071 workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp);
2073 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp);
2076 * This should never happen since we only create pagedeps
2077 * with the vnode lock held. Could be an assert.
2079 WORKITEM_FREE(pagedep, D_PAGEDEP);
2082 pagedep->pd_ino = ino;
2083 pagedep->pd_lbn = lbn;
2084 LIST_INIT(&pagedep->pd_dirremhd);
2085 LIST_INIT(&pagedep->pd_pendinghd);
2086 for (i = 0; i < DAHASHSZ; i++)
2087 LIST_INIT(&pagedep->pd_diraddhd[i]);
2088 LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash);
2089 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list);
2090 *pagedeppp = pagedep;
2095 * Structures and routines associated with inodedep caching.
2097 LIST_HEAD(inodedep_hashhead, inodedep) *inodedep_hashtbl;
2098 static u_long inodedep_hash; /* size of hash table - 1 */
2099 #define INODEDEP_HASH(fs, inum) \
2100 (&inodedep_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & inodedep_hash])
2103 inodedep_find(inodedephd, fs, inum, inodedeppp)
2104 struct inodedep_hashhead *inodedephd;
2107 struct inodedep **inodedeppp;
2109 struct inodedep *inodedep;
2111 LIST_FOREACH(inodedep, inodedephd, id_hash)
2112 if (inum == inodedep->id_ino && fs == inodedep->id_fs)
2115 *inodedeppp = inodedep;
2123 * Look up an inodedep. Return 1 if found, 0 if not found.
2124 * If not found, allocate if DEPALLOC flag is passed.
2125 * Found or allocated entry is returned in inodedeppp.
2126 * This routine must be called with splbio interrupts blocked.
2129 inodedep_lookup(mp, inum, flags, inodedeppp)
2133 struct inodedep **inodedeppp;
2135 struct inodedep *inodedep;
2136 struct inodedep_hashhead *inodedephd;
2139 mtx_assert(&lk, MA_OWNED);
2140 fs = VFSTOUFS(mp)->um_fs;
2141 inodedephd = INODEDEP_HASH(fs, inum);
2143 if (inodedep_find(inodedephd, fs, inum, inodedeppp))
2145 if ((flags & DEPALLOC) == 0)
2148 * If we are over our limit, try to improve the situation.
2150 if (dep_current[D_INODEDEP] > max_softdeps && (flags & NODELAY) == 0)
2151 request_cleanup(mp, FLUSH_INODES);
2153 inodedep = malloc(sizeof(struct inodedep),
2154 M_INODEDEP, M_SOFTDEP_FLAGS);
2155 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp);
2157 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) {
2158 WORKITEM_FREE(inodedep, D_INODEDEP);
2161 inodedep->id_fs = fs;
2162 inodedep->id_ino = inum;
2163 inodedep->id_state = ALLCOMPLETE;
2164 inodedep->id_nlinkdelta = 0;
2165 inodedep->id_savedino1 = NULL;
2166 inodedep->id_savedsize = -1;
2167 inodedep->id_savedextsize = -1;
2168 inodedep->id_savednlink = -1;
2169 inodedep->id_bmsafemap = NULL;
2170 inodedep->id_mkdiradd = NULL;
2171 LIST_INIT(&inodedep->id_dirremhd);
2172 LIST_INIT(&inodedep->id_pendinghd);
2173 LIST_INIT(&inodedep->id_inowait);
2174 LIST_INIT(&inodedep->id_bufwait);
2175 TAILQ_INIT(&inodedep->id_inoreflst);
2176 TAILQ_INIT(&inodedep->id_inoupdt);
2177 TAILQ_INIT(&inodedep->id_newinoupdt);
2178 TAILQ_INIT(&inodedep->id_extupdt);
2179 TAILQ_INIT(&inodedep->id_newextupdt);
2180 TAILQ_INIT(&inodedep->id_freeblklst);
2181 LIST_INSERT_HEAD(inodedephd, inodedep, id_hash);
2182 *inodedeppp = inodedep;
2187 * Structures and routines associated with newblk caching.
2189 LIST_HEAD(newblk_hashhead, newblk) *newblk_hashtbl;
2190 u_long newblk_hash; /* size of hash table - 1 */
2191 #define NEWBLK_HASH(fs, inum) \
2192 (&newblk_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & newblk_hash])
2195 newblk_find(newblkhd, mp, newblkno, flags, newblkpp)
2196 struct newblk_hashhead *newblkhd;
2198 ufs2_daddr_t newblkno;
2200 struct newblk **newblkpp;
2202 struct newblk *newblk;
2204 LIST_FOREACH(newblk, newblkhd, nb_hash) {
2205 if (newblkno != newblk->nb_newblkno)
2207 if (mp != newblk->nb_list.wk_mp)
2210 * If we're creating a new dependency don't match those that
2211 * have already been converted to allocdirects. This is for
2214 if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK)
2227 * Look up a newblk. Return 1 if found, 0 if not found.
2228 * If not found, allocate if DEPALLOC flag is passed.
2229 * Found or allocated entry is returned in newblkpp.
2232 newblk_lookup(mp, newblkno, flags, newblkpp)
2234 ufs2_daddr_t newblkno;
2236 struct newblk **newblkpp;
2238 struct newblk *newblk;
2239 struct newblk_hashhead *newblkhd;
2241 newblkhd = NEWBLK_HASH(VFSTOUFS(mp)->um_fs, newblkno);
2242 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp))
2244 if ((flags & DEPALLOC) == 0)
2247 newblk = malloc(sizeof(union allblk), M_NEWBLK,
2248 M_SOFTDEP_FLAGS | M_ZERO);
2249 workitem_alloc(&newblk->nb_list, D_NEWBLK, mp);
2251 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp)) {
2252 WORKITEM_FREE(newblk, D_NEWBLK);
2255 newblk->nb_freefrag = NULL;
2256 LIST_INIT(&newblk->nb_indirdeps);
2257 LIST_INIT(&newblk->nb_newdirblk);
2258 LIST_INIT(&newblk->nb_jwork);
2259 newblk->nb_state = ATTACHED;
2260 newblk->nb_newblkno = newblkno;
2261 LIST_INSERT_HEAD(newblkhd, newblk, nb_hash);
2267 * Structures and routines associated with freed indirect block caching.
2269 struct freeworklst *indir_hashtbl;
2270 u_long indir_hash; /* size of hash table - 1 */
2271 #define INDIR_HASH(mp, blkno) \
2272 (&indir_hashtbl[((((register_t)(mp)) >> 13) + (blkno)) & indir_hash])
2275 * Lookup an indirect block in the indir hash table. The freework is
2276 * removed and potentially freed. The caller must do a blocking journal
2277 * write before writing to the blkno.
2280 indirblk_lookup(mp, blkno)
2284 struct freework *freework;
2285 struct freeworklst *wkhd;
2287 wkhd = INDIR_HASH(mp, blkno);
2288 TAILQ_FOREACH(freework, wkhd, fw_next) {
2289 if (freework->fw_blkno != blkno)
2291 if (freework->fw_list.wk_mp != mp)
2293 indirblk_remove(freework);
2300 * Insert an indirect block represented by freework into the indirblk
2301 * hash table so that it may prevent the block from being re-used prior
2302 * to the journal being written.
2305 indirblk_insert(freework)
2306 struct freework *freework;
2308 struct jblocks *jblocks;
2311 jblocks = VFSTOUFS(freework->fw_list.wk_mp)->softdep_jblocks;
2312 jseg = TAILQ_LAST(&jblocks->jb_segs, jseglst);
2316 LIST_INSERT_HEAD(&jseg->js_indirs, freework, fw_segs);
2317 TAILQ_INSERT_HEAD(INDIR_HASH(freework->fw_list.wk_mp,
2318 freework->fw_blkno), freework, fw_next);
2319 freework->fw_state &= ~DEPCOMPLETE;
2323 indirblk_remove(freework)
2324 struct freework *freework;
2327 LIST_REMOVE(freework, fw_segs);
2328 TAILQ_REMOVE(INDIR_HASH(freework->fw_list.wk_mp,
2329 freework->fw_blkno), freework, fw_next);
2330 freework->fw_state |= DEPCOMPLETE;
2331 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE)
2332 WORKITEM_FREE(freework, D_FREEWORK);
2336 * Executed during filesystem system initialization before
2337 * mounting any filesystems.
2340 softdep_initialize()
2344 LIST_INIT(&mkdirlisthd);
2345 max_softdeps = desiredvnodes * 4;
2346 pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP, &pagedep_hash);
2347 inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, &inodedep_hash);
2348 newblk_hashtbl = hashinit(desiredvnodes / 5, M_NEWBLK, &newblk_hash);
2349 bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP, &bmsafemap_hash);
2350 i = 1 << (ffs(desiredvnodes / 10) - 1);
2351 indir_hashtbl = malloc(i * sizeof(indir_hashtbl[0]), M_FREEWORK,
2354 for (i = 0; i <= indir_hash; i++)
2355 TAILQ_INIT(&indir_hashtbl[i]);
2357 /* initialise bioops hack */
2358 bioops.io_start = softdep_disk_io_initiation;
2359 bioops.io_complete = softdep_disk_write_complete;
2360 bioops.io_deallocate = softdep_deallocate_dependencies;
2361 bioops.io_countdeps = softdep_count_dependencies;
2363 /* Initialize the callout with an mtx. */
2364 callout_init_mtx(&softdep_callout, &lk, 0);
2368 * Executed after all filesystems have been unmounted during
2369 * filesystem module unload.
2372 softdep_uninitialize()
2375 callout_drain(&softdep_callout);
2376 hashdestroy(pagedep_hashtbl, M_PAGEDEP, pagedep_hash);
2377 hashdestroy(inodedep_hashtbl, M_INODEDEP, inodedep_hash);
2378 hashdestroy(newblk_hashtbl, M_NEWBLK, newblk_hash);
2379 hashdestroy(bmsafemap_hashtbl, M_BMSAFEMAP, bmsafemap_hash);
2380 free(indir_hashtbl, M_FREEWORK);
2384 * Called at mount time to notify the dependency code that a
2385 * filesystem wishes to use it.
2388 softdep_mount(devvp, mp, fs, cred)
2389 struct vnode *devvp;
2394 struct csum_total cstotal;
2395 struct ufsmount *ump;
2401 mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP;
2402 if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) {
2403 mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) |
2404 MNTK_SOFTDEP | MNTK_NOASYNC;
2408 LIST_INIT(&ump->softdep_workitem_pending);
2409 LIST_INIT(&ump->softdep_journal_pending);
2410 TAILQ_INIT(&ump->softdep_unlinked);
2411 LIST_INIT(&ump->softdep_dirtycg);
2412 ump->softdep_worklist_tail = NULL;
2413 ump->softdep_on_worklist = 0;
2414 ump->softdep_deps = 0;
2415 if ((fs->fs_flags & FS_SUJ) &&
2416 (error = journal_mount(mp, fs, cred)) != 0) {
2417 printf("Failed to start journal: %d\n", error);
2421 * When doing soft updates, the counters in the
2422 * superblock may have gotten out of sync. Recomputation
2423 * can take a long time and can be deferred for background
2424 * fsck. However, the old behavior of scanning the cylinder
2425 * groups and recalculating them at mount time is available
2426 * by setting vfs.ffs.compute_summary_at_mount to one.
2428 if (compute_summary_at_mount == 0 || fs->fs_clean != 0)
2430 bzero(&cstotal, sizeof cstotal);
2431 for (cyl = 0; cyl < fs->fs_ncg; cyl++) {
2432 if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)),
2433 fs->fs_cgsize, cred, &bp)) != 0) {
2437 cgp = (struct cg *)bp->b_data;
2438 cstotal.cs_nffree += cgp->cg_cs.cs_nffree;
2439 cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree;
2440 cstotal.cs_nifree += cgp->cg_cs.cs_nifree;
2441 cstotal.cs_ndir += cgp->cg_cs.cs_ndir;
2442 fs->fs_cs(fs, cyl) = cgp->cg_cs;
2446 if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal))
2447 printf("%s: superblock summary recomputed\n", fs->fs_fsmnt);
2449 bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal);
2459 mp->mnt_flag &= ~MNT_SOFTDEP;
2460 if (MOUNTEDSUJ(mp) == 0) {
2464 mp->mnt_flag &= ~MNT_SUJ;
2466 journal_unmount(mp);
2469 static struct jblocks *
2470 jblocks_create(void)
2472 struct jblocks *jblocks;
2474 jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO);
2475 TAILQ_INIT(&jblocks->jb_segs);
2476 jblocks->jb_avail = 10;
2477 jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail,
2478 M_JBLOCKS, M_WAITOK | M_ZERO);
2484 jblocks_alloc(jblocks, bytes, actual)
2485 struct jblocks *jblocks;
2490 struct jextent *jext;
2494 blocks = bytes / DEV_BSIZE;
2495 jext = &jblocks->jb_extent[jblocks->jb_head];
2496 freecnt = jext->je_blocks - jblocks->jb_off;
2498 jblocks->jb_off = 0;
2499 if (++jblocks->jb_head > jblocks->jb_used)
2500 jblocks->jb_head = 0;
2501 jext = &jblocks->jb_extent[jblocks->jb_head];
2502 freecnt = jext->je_blocks;
2504 if (freecnt > blocks)
2506 *actual = freecnt * DEV_BSIZE;
2507 daddr = jext->je_daddr + jblocks->jb_off;
2508 jblocks->jb_off += freecnt;
2509 jblocks->jb_free -= freecnt;
2515 jblocks_free(jblocks, mp, bytes)
2516 struct jblocks *jblocks;
2521 jblocks->jb_free += bytes / DEV_BSIZE;
2522 if (jblocks->jb_suspended)
2528 jblocks_destroy(jblocks)
2529 struct jblocks *jblocks;
2532 if (jblocks->jb_extent)
2533 free(jblocks->jb_extent, M_JBLOCKS);
2534 free(jblocks, M_JBLOCKS);
2538 jblocks_add(jblocks, daddr, blocks)
2539 struct jblocks *jblocks;
2543 struct jextent *jext;
2545 jblocks->jb_blocks += blocks;
2546 jblocks->jb_free += blocks;
2547 jext = &jblocks->jb_extent[jblocks->jb_used];
2548 /* Adding the first block. */
2549 if (jext->je_daddr == 0) {
2550 jext->je_daddr = daddr;
2551 jext->je_blocks = blocks;
2554 /* Extending the last extent. */
2555 if (jext->je_daddr + jext->je_blocks == daddr) {
2556 jext->je_blocks += blocks;
2559 /* Adding a new extent. */
2560 if (++jblocks->jb_used == jblocks->jb_avail) {
2561 jblocks->jb_avail *= 2;
2562 jext = malloc(sizeof(struct jextent) * jblocks->jb_avail,
2563 M_JBLOCKS, M_WAITOK | M_ZERO);
2564 memcpy(jext, jblocks->jb_extent,
2565 sizeof(struct jextent) * jblocks->jb_used);
2566 free(jblocks->jb_extent, M_JBLOCKS);
2567 jblocks->jb_extent = jext;
2569 jext = &jblocks->jb_extent[jblocks->jb_used];
2570 jext->je_daddr = daddr;
2571 jext->je_blocks = blocks;
2576 softdep_journal_lookup(mp, vpp)
2580 struct componentname cnp;
2585 error = VFS_VGET(mp, ROOTINO, LK_EXCLUSIVE, &dvp);
2588 bzero(&cnp, sizeof(cnp));
2589 cnp.cn_nameiop = LOOKUP;
2590 cnp.cn_flags = ISLASTCN;
2591 cnp.cn_thread = curthread;
2592 cnp.cn_cred = curthread->td_ucred;
2593 cnp.cn_pnbuf = SUJ_FILE;
2594 cnp.cn_nameptr = SUJ_FILE;
2595 cnp.cn_namelen = strlen(SUJ_FILE);
2596 error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal);
2600 error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp);
2605 * Open and verify the journal file.
2608 journal_mount(mp, fs, cred)
2613 struct jblocks *jblocks;
2621 error = softdep_journal_lookup(mp, &vp);
2623 printf("Failed to find journal. Use tunefs to create one\n");
2627 if (ip->i_size < SUJ_MIN) {
2631 bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */
2632 jblocks = jblocks_create();
2633 for (i = 0; i < bcount; i++) {
2634 error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL);
2637 jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag));
2640 jblocks_destroy(jblocks);
2643 jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */
2644 jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */
2645 VFSTOUFS(mp)->softdep_jblocks = jblocks;
2649 mp->mnt_flag |= MNT_SUJ;
2650 mp->mnt_flag &= ~MNT_SOFTDEP;
2653 * Only validate the journal contents if the
2654 * filesystem is clean, otherwise we write the logs
2655 * but they'll never be used. If the filesystem was
2656 * still dirty when we mounted it the journal is
2657 * invalid and a new journal can only be valid if it
2658 * starts from a clean mount.
2661 DIP_SET(ip, i_modrev, fs->fs_mtime);
2662 ip->i_flags |= IN_MODIFIED;
2674 struct ufsmount *ump;
2677 if (ump->softdep_jblocks)
2678 jblocks_destroy(ump->softdep_jblocks);
2679 ump->softdep_jblocks = NULL;
2683 * Called when a journal record is ready to be written. Space is allocated
2684 * and the journal entry is created when the journal is flushed to stable
2689 struct worklist *wk;
2691 struct ufsmount *ump;
2693 mtx_assert(&lk, MA_OWNED);
2694 ump = VFSTOUFS(wk->wk_mp);
2695 if (wk->wk_state & ONWORKLIST)
2696 panic("add_to_journal: %s(0x%X) already on list",
2697 TYPENAME(wk->wk_type), wk->wk_state);
2698 wk->wk_state |= ONWORKLIST | DEPCOMPLETE;
2699 if (LIST_EMPTY(&ump->softdep_journal_pending)) {
2700 ump->softdep_jblocks->jb_age = ticks;
2701 LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list);
2703 LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list);
2704 ump->softdep_journal_tail = wk;
2705 ump->softdep_on_journal += 1;
2709 * Remove an arbitrary item for the journal worklist maintain the tail
2710 * pointer. This happens when a new operation obviates the need to
2711 * journal an old operation.
2714 remove_from_journal(wk)
2715 struct worklist *wk;
2717 struct ufsmount *ump;
2719 mtx_assert(&lk, MA_OWNED);
2720 ump = VFSTOUFS(wk->wk_mp);
2723 struct worklist *wkn;
2725 LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list)
2729 panic("remove_from_journal: %p is not in journal", wk);
2733 * We emulate a TAILQ to save space in most structures which do not
2734 * require TAILQ semantics. Here we must update the tail position
2735 * when removing the tail which is not the final entry. This works
2736 * only if the worklist linkage are at the beginning of the structure.
2738 if (ump->softdep_journal_tail == wk)
2739 ump->softdep_journal_tail =
2740 (struct worklist *)wk->wk_list.le_prev;
2742 WORKLIST_REMOVE(wk);
2743 ump->softdep_on_journal -= 1;
2747 * Check for journal space as well as dependency limits so the prelink
2748 * code can throttle both journaled and non-journaled filesystems.
2749 * Threshold is 0 for low and 1 for min.
2752 journal_space(ump, thresh)
2753 struct ufsmount *ump;
2756 struct jblocks *jblocks;
2759 jblocks = ump->softdep_jblocks;
2760 if (jblocks == NULL)
2763 * We use a tighter restriction here to prevent request_cleanup()
2764 * running in threads from running into locks we currently hold.
2766 if (dep_current[D_INODEDEP] > (max_softdeps / 10) * 9)
2769 thresh = jblocks->jb_min;
2771 thresh = jblocks->jb_low;
2772 avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE;
2773 avail = jblocks->jb_free - avail;
2775 return (avail > thresh);
2779 journal_suspend(ump)
2780 struct ufsmount *ump;
2782 struct jblocks *jblocks;
2786 jblocks = ump->softdep_jblocks;
2788 if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) {
2790 mp->mnt_kern_flag |= MNTK_SUSPEND;
2791 mp->mnt_susp_owner = FIRST_THREAD_IN_PROC(softdepproc);
2793 jblocks->jb_suspended = 1;
2798 journal_unsuspend(struct ufsmount *ump)
2800 struct jblocks *jblocks;
2804 jblocks = ump->softdep_jblocks;
2806 if (jblocks != NULL && jblocks->jb_suspended &&
2807 journal_space(ump, jblocks->jb_min)) {
2808 jblocks->jb_suspended = 0;
2810 mp->mnt_susp_owner = curthread;
2811 vfs_write_resume(mp);
2819 * Called before any allocation function to be certain that there is
2820 * sufficient space in the journal prior to creating any new records.
2821 * Since in the case of block allocation we may have multiple locked
2822 * buffers at the time of the actual allocation we can not block
2823 * when the journal records are created. Doing so would create a deadlock
2824 * if any of these buffers needed to be flushed to reclaim space. Instead
2825 * we require a sufficiently large amount of available space such that
2826 * each thread in the system could have passed this allocation check and
2827 * still have sufficient free space. With 20% of a minimum journal size
2828 * of 1MB we have 6553 records available.
2831 softdep_prealloc(vp, waitok)
2835 struct ufsmount *ump;
2838 * Nothing to do if we are not running journaled soft updates.
2839 * If we currently hold the snapshot lock, we must avoid handling
2840 * other resources that could cause deadlock.
2842 if (DOINGSUJ(vp) == 0 || IS_SNAPSHOT(VTOI(vp)))
2844 ump = VFSTOUFS(vp->v_mount);
2846 if (journal_space(ump, 0)) {
2852 if (waitok == MNT_NOWAIT)
2855 * Attempt to sync this vnode once to flush any journal
2856 * work attached to it.
2858 if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0)
2859 ffs_syncvnode(vp, waitok, 0);
2861 process_removes(vp);
2862 process_truncates(vp);
2863 if (journal_space(ump, 0) == 0) {
2865 if (journal_space(ump, 1) == 0)
2866 journal_suspend(ump);
2874 * Before adjusting a link count on a vnode verify that we have sufficient
2875 * journal space. If not, process operations that depend on the currently
2876 * locked pair of vnodes to try to flush space as the syncer, buf daemon,
2877 * and softdep flush threads can not acquire these locks to reclaim space.
2880 softdep_prelink(dvp, vp)
2884 struct ufsmount *ump;
2886 ump = VFSTOUFS(dvp->v_mount);
2887 mtx_assert(&lk, MA_OWNED);
2889 * Nothing to do if we have sufficient journal space.
2890 * If we currently hold the snapshot lock, we must avoid
2891 * handling other resources that could cause deadlock.
2893 if (journal_space(ump, 0) || (vp && IS_SNAPSHOT(VTOI(vp))))
2898 ffs_syncvnode(vp, MNT_NOWAIT, 0);
2899 ffs_syncvnode(dvp, MNT_WAIT, 0);
2901 /* Process vp before dvp as it may create .. removes. */
2903 process_removes(vp);
2904 process_truncates(vp);
2906 process_removes(dvp);
2907 process_truncates(dvp);
2909 process_worklist_item(UFSTOVFS(ump), 2, LK_NOWAIT);
2910 if (journal_space(ump, 0) == 0) {
2912 if (journal_space(ump, 1) == 0)
2913 journal_suspend(ump);
2918 jseg_write(ump, jseg, data)
2919 struct ufsmount *ump;
2923 struct jsegrec *rec;
2925 rec = (struct jsegrec *)data;
2926 rec->jsr_seq = jseg->js_seq;
2927 rec->jsr_oldest = jseg->js_oldseq;
2928 rec->jsr_cnt = jseg->js_cnt;
2929 rec->jsr_blocks = jseg->js_size / ump->um_devvp->v_bufobj.bo_bsize;
2931 rec->jsr_time = ump->um_fs->fs_mtime;
2935 inoref_write(inoref, jseg, rec)
2936 struct inoref *inoref;
2938 struct jrefrec *rec;
2941 inoref->if_jsegdep->jd_seg = jseg;
2942 rec->jr_ino = inoref->if_ino;
2943 rec->jr_parent = inoref->if_parent;
2944 rec->jr_nlink = inoref->if_nlink;
2945 rec->jr_mode = inoref->if_mode;
2946 rec->jr_diroff = inoref->if_diroff;
2950 jaddref_write(jaddref, jseg, data)
2951 struct jaddref *jaddref;
2955 struct jrefrec *rec;
2957 rec = (struct jrefrec *)data;
2958 rec->jr_op = JOP_ADDREF;
2959 inoref_write(&jaddref->ja_ref, jseg, rec);
2963 jremref_write(jremref, jseg, data)
2964 struct jremref *jremref;
2968 struct jrefrec *rec;
2970 rec = (struct jrefrec *)data;
2971 rec->jr_op = JOP_REMREF;
2972 inoref_write(&jremref->jr_ref, jseg, rec);
2976 jmvref_write(jmvref, jseg, data)
2977 struct jmvref *jmvref;
2983 rec = (struct jmvrec *)data;
2984 rec->jm_op = JOP_MVREF;
2985 rec->jm_ino = jmvref->jm_ino;
2986 rec->jm_parent = jmvref->jm_parent;
2987 rec->jm_oldoff = jmvref->jm_oldoff;
2988 rec->jm_newoff = jmvref->jm_newoff;
2992 jnewblk_write(jnewblk, jseg, data)
2993 struct jnewblk *jnewblk;
2997 struct jblkrec *rec;
2999 jnewblk->jn_jsegdep->jd_seg = jseg;
3000 rec = (struct jblkrec *)data;
3001 rec->jb_op = JOP_NEWBLK;
3002 rec->jb_ino = jnewblk->jn_ino;
3003 rec->jb_blkno = jnewblk->jn_blkno;
3004 rec->jb_lbn = jnewblk->jn_lbn;
3005 rec->jb_frags = jnewblk->jn_frags;
3006 rec->jb_oldfrags = jnewblk->jn_oldfrags;
3010 jfreeblk_write(jfreeblk, jseg, data)
3011 struct jfreeblk *jfreeblk;
3015 struct jblkrec *rec;
3017 jfreeblk->jf_dep.jb_jsegdep->jd_seg = jseg;
3018 rec = (struct jblkrec *)data;
3019 rec->jb_op = JOP_FREEBLK;
3020 rec->jb_ino = jfreeblk->jf_ino;
3021 rec->jb_blkno = jfreeblk->jf_blkno;
3022 rec->jb_lbn = jfreeblk->jf_lbn;
3023 rec->jb_frags = jfreeblk->jf_frags;
3024 rec->jb_oldfrags = 0;
3028 jfreefrag_write(jfreefrag, jseg, data)
3029 struct jfreefrag *jfreefrag;
3033 struct jblkrec *rec;
3035 jfreefrag->fr_jsegdep->jd_seg = jseg;
3036 rec = (struct jblkrec *)data;
3037 rec->jb_op = JOP_FREEBLK;
3038 rec->jb_ino = jfreefrag->fr_ino;
3039 rec->jb_blkno = jfreefrag->fr_blkno;
3040 rec->jb_lbn = jfreefrag->fr_lbn;
3041 rec->jb_frags = jfreefrag->fr_frags;
3042 rec->jb_oldfrags = 0;
3046 jtrunc_write(jtrunc, jseg, data)
3047 struct jtrunc *jtrunc;
3051 struct jtrncrec *rec;
3053 jtrunc->jt_dep.jb_jsegdep->jd_seg = jseg;
3054 rec = (struct jtrncrec *)data;
3055 rec->jt_op = JOP_TRUNC;
3056 rec->jt_ino = jtrunc->jt_ino;
3057 rec->jt_size = jtrunc->jt_size;
3058 rec->jt_extsize = jtrunc->jt_extsize;
3062 jfsync_write(jfsync, jseg, data)
3063 struct jfsync *jfsync;
3067 struct jtrncrec *rec;
3069 rec = (struct jtrncrec *)data;
3070 rec->jt_op = JOP_SYNC;
3071 rec->jt_ino = jfsync->jfs_ino;
3072 rec->jt_size = jfsync->jfs_size;
3073 rec->jt_extsize = jfsync->jfs_extsize;
3077 softdep_flushjournal(mp)
3080 struct jblocks *jblocks;
3081 struct ufsmount *ump;
3083 if (MOUNTEDSUJ(mp) == 0)
3086 jblocks = ump->softdep_jblocks;
3088 while (ump->softdep_on_journal) {
3089 jblocks->jb_needseg = 1;
3090 softdep_process_journal(mp, NULL, MNT_WAIT);
3095 static void softdep_synchronize_completed(struct bio *);
3096 static void softdep_synchronize(struct bio *, struct ufsmount *, void *);
3099 softdep_synchronize_completed(bp)
3102 struct jseg *oldest;
3106 * caller1 marks the last segment written before we issued the
3107 * synchronize cache.
3109 jseg = bp->bio_caller1;
3113 * Mark all the journal entries waiting on the synchronize cache
3114 * as completed so they may continue on.
3116 while (jseg != NULL && (jseg->js_state & COMPLETE) == 0) {
3117 jseg->js_state |= COMPLETE;
3119 jseg = TAILQ_PREV(jseg, jseglst, js_next);
3122 * Restart deferred journal entry processing from the oldest
3126 complete_jsegs(oldest);
3133 * Send BIO_FLUSH/SYNCHRONIZE CACHE to the device to enforce write ordering
3134 * barriers. The journal must be written prior to any blocks that depend
3135 * on it and the journal can not be released until the blocks have be
3136 * written. This code handles both barriers simultaneously.
3139 softdep_synchronize(bp, ump, caller1)
3141 struct ufsmount *ump;
3145 bp->bio_cmd = BIO_FLUSH;
3146 bp->bio_flags |= BIO_ORDERED;
3147 bp->bio_data = NULL;
3148 bp->bio_offset = ump->um_cp->provider->mediasize;
3150 bp->bio_done = softdep_synchronize_completed;
3151 bp->bio_caller1 = caller1;
3153 (struct g_consumer *)ump->um_devvp->v_bufobj.bo_private);
3157 * Flush some journal records to disk.
3160 softdep_process_journal(mp, needwk, flags)
3162 struct worklist *needwk;
3165 struct jblocks *jblocks;
3166 struct ufsmount *ump;
3167 struct worklist *wk;
3175 int jrecmin; /* Minimum records per block. */
3176 int jrecmax; /* Maximum records per block. */
3182 if (MOUNTEDSUJ(mp) == 0)
3184 shouldflush = softdep_flushcache;
3189 jblocks = ump->softdep_jblocks;
3190 devbsize = ump->um_devvp->v_bufobj.bo_bsize;
3192 * We write anywhere between a disk block and fs block. The upper
3193 * bound is picked to prevent buffer cache fragmentation and limit
3194 * processing time per I/O.
3196 jrecmin = (devbsize / JREC_SIZE) - 1; /* -1 for seg header */
3197 jrecmax = (fs->fs_bsize / devbsize) * jrecmin;
3200 cnt = ump->softdep_on_journal;
3202 * Criteria for writing a segment:
3203 * 1) We have a full block.
3204 * 2) We're called from jwait() and haven't found the
3206 * 3) Always write if needseg is set.
3207 * 4) If we are called from process_worklist and have
3208 * not yet written anything we write a partial block
3209 * to enforce a 1 second maximum latency on journal
3212 if (cnt < (jrecmax - 1) && needwk == NULL &&
3213 jblocks->jb_needseg == 0 && (segwritten || cnt == 0))
3217 * Verify some free journal space. softdep_prealloc() should
3218 * guarantee that we don't run out so this is indicative of
3219 * a problem with the flow control. Try to recover
3220 * gracefully in any event.
3222 while (jblocks->jb_free == 0) {
3223 if (flags != MNT_WAIT)
3225 printf("softdep: Out of journal space!\n");
3227 msleep(jblocks, &lk, PRIBIO, "jblocks", hz);
3230 jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS);
3231 workitem_alloc(&jseg->js_list, D_JSEG, mp);
3232 LIST_INIT(&jseg->js_entries);
3233 LIST_INIT(&jseg->js_indirs);
3234 jseg->js_state = ATTACHED;
3235 if (shouldflush == 0)
3236 jseg->js_state |= COMPLETE;
3237 else if (bio == NULL)
3238 bio = g_alloc_bio();
3239 jseg->js_jblocks = jblocks;
3240 bp = geteblk(fs->fs_bsize, 0);
3243 * If there was a race while we were allocating the block
3244 * and jseg the entry we care about was likely written.
3245 * We bail out in both the WAIT and NOWAIT case and assume
3246 * the caller will loop if the entry it cares about is
3249 cnt = ump->softdep_on_journal;
3250 if (cnt + jblocks->jb_needseg == 0 || jblocks->jb_free == 0) {
3251 bp->b_flags |= B_INVAL | B_NOCACHE;
3252 WORKITEM_FREE(jseg, D_JSEG);
3259 * Calculate the disk block size required for the available
3260 * records rounded to the min size.
3264 else if (cnt < jrecmax)
3265 size = howmany(cnt, jrecmin) * devbsize;
3267 size = fs->fs_bsize;
3269 * Allocate a disk block for this journal data and account
3270 * for truncation of the requested size if enough contiguous
3271 * space was not available.
3273 bp->b_blkno = jblocks_alloc(jblocks, size, &size);
3274 bp->b_lblkno = bp->b_blkno;
3275 bp->b_offset = bp->b_blkno * DEV_BSIZE;
3276 bp->b_bcount = size;
3277 bp->b_bufobj = &ump->um_devvp->v_bufobj;
3278 bp->b_flags &= ~B_INVAL;
3279 bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY;
3281 * Initialize our jseg with cnt records. Assign the next
3282 * sequence number to it and link it in-order.
3284 cnt = MIN(cnt, (size / devbsize) * jrecmin);
3287 jseg->js_refs = cnt + 1; /* Self ref. */
3288 jseg->js_size = size;
3289 jseg->js_seq = jblocks->jb_nextseq++;
3290 if (jblocks->jb_oldestseg == NULL)
3291 jblocks->jb_oldestseg = jseg;
3292 jseg->js_oldseq = jblocks->jb_oldestseg->js_seq;
3293 TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next);
3294 if (jblocks->jb_writeseg == NULL)
3295 jblocks->jb_writeseg = jseg;
3297 * Start filling in records from the pending list.
3301 while ((wk = LIST_FIRST(&ump->softdep_journal_pending))
3305 /* Place a segment header on every device block. */
3306 if ((off % devbsize) == 0) {
3307 jseg_write(ump, jseg, data);
3309 data = bp->b_data + off;
3313 remove_from_journal(wk);
3314 wk->wk_state |= INPROGRESS;
3315 WORKLIST_INSERT(&jseg->js_entries, wk);
3316 switch (wk->wk_type) {
3318 jaddref_write(WK_JADDREF(wk), jseg, data);
3321 jremref_write(WK_JREMREF(wk), jseg, data);
3324 jmvref_write(WK_JMVREF(wk), jseg, data);
3327 jnewblk_write(WK_JNEWBLK(wk), jseg, data);
3330 jfreeblk_write(WK_JFREEBLK(wk), jseg, data);
3333 jfreefrag_write(WK_JFREEFRAG(wk), jseg, data);
3336 jtrunc_write(WK_JTRUNC(wk), jseg, data);
3339 jfsync_write(WK_JFSYNC(wk), jseg, data);
3342 panic("process_journal: Unknown type %s",
3343 TYPENAME(wk->wk_type));
3347 data = bp->b_data + off;
3351 * Write this one buffer and continue.
3354 jblocks->jb_needseg = 0;
3355 WORKLIST_INSERT(&bp->b_dep, &jseg->js_list);
3357 BO_LOCK(bp->b_bufobj);
3358 bgetvp(ump->um_devvp, bp);
3359 BO_UNLOCK(bp->b_bufobj);
3361 * We only do the blocking wait once we find the journal
3362 * entry we're looking for.
3364 if (needwk == NULL && flags == MNT_WAIT)
3371 * If we wrote a segment issue a synchronize cache so the journal
3372 * is reflected on disk before the data is written. Since reclaiming
3373 * journal space also requires writing a journal record this
3374 * process also enforces a barrier before reclamation.
3376 if (segwritten && shouldflush) {
3377 softdep_synchronize(bio, ump,
3378 TAILQ_LAST(&jblocks->jb_segs, jseglst));
3382 * If we've suspended the filesystem because we ran out of journal
3383 * space either try to sync it here to make some progress or
3384 * unsuspend it if we already have.
3386 if (flags == 0 && jblocks->jb_suspended) {
3387 if (journal_unsuspend(ump))
3390 VFS_SYNC(mp, MNT_NOWAIT);
3391 ffs_sbupdate(ump, MNT_WAIT, 0);
3397 * Complete a jseg, allowing all dependencies awaiting journal writes
3398 * to proceed. Each journal dependency also attaches a jsegdep to dependent
3399 * structures so that the journal segment can be freed to reclaim space.
3405 struct worklist *wk;
3406 struct jmvref *jmvref;
3412 while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) {
3413 WORKLIST_REMOVE(wk);
3414 waiting = wk->wk_state & IOWAITING;
3415 wk->wk_state &= ~(INPROGRESS | IOWAITING);
3416 wk->wk_state |= COMPLETE;
3417 KASSERT(i++ < jseg->js_cnt,
3418 ("handle_written_jseg: overflow %d >= %d",
3419 i - 1, jseg->js_cnt));
3420 switch (wk->wk_type) {
3422 handle_written_jaddref(WK_JADDREF(wk));
3425 handle_written_jremref(WK_JREMREF(wk));
3428 rele_jseg(jseg); /* No jsegdep. */
3429 jmvref = WK_JMVREF(wk);
3430 LIST_REMOVE(jmvref, jm_deps);
3431 if ((jmvref->jm_pagedep->pd_state & ONWORKLIST) == 0)
3432 free_pagedep(jmvref->jm_pagedep);
3433 WORKITEM_FREE(jmvref, D_JMVREF);
3436 handle_written_jnewblk(WK_JNEWBLK(wk));
3439 handle_written_jblkdep(&WK_JFREEBLK(wk)->jf_dep);
3442 handle_written_jblkdep(&WK_JTRUNC(wk)->jt_dep);
3445 rele_jseg(jseg); /* No jsegdep. */
3446 WORKITEM_FREE(wk, D_JFSYNC);
3449 handle_written_jfreefrag(WK_JFREEFRAG(wk));
3452 panic("handle_written_jseg: Unknown type %s",
3453 TYPENAME(wk->wk_type));
3459 /* Release the self reference so the structure may be freed. */
3464 * Determine which jsegs are ready for completion processing. Waits for
3465 * synchronize cache to complete as well as forcing in-order completion
3466 * of journal entries.
3469 complete_jsegs(jseg)
3472 struct jblocks *jblocks;
3475 jblocks = jseg->js_jblocks;
3477 * Don't allow out of order completions. If this isn't the first
3478 * block wait for it to write before we're done.
3480 if (jseg != jblocks->jb_writeseg)
3482 /* Iterate through available jsegs processing their entries. */
3483 while (jseg && (jseg->js_state & ALLCOMPLETE) == ALLCOMPLETE) {
3484 jblocks->jb_oldestwrseq = jseg->js_oldseq;
3485 jsegn = TAILQ_NEXT(jseg, js_next);
3486 complete_jseg(jseg);
3489 jblocks->jb_writeseg = jseg;
3491 * Attempt to free jsegs now that oldestwrseq may have advanced.
3493 free_jsegs(jblocks);
3497 * Mark a jseg as DEPCOMPLETE and throw away the buffer. Attempt to handle
3498 * the final completions.
3501 handle_written_jseg(jseg, bp)
3506 if (jseg->js_refs == 0)
3507 panic("handle_written_jseg: No self-reference on %p", jseg);
3508 jseg->js_state |= DEPCOMPLETE;
3510 * We'll never need this buffer again, set flags so it will be
3513 bp->b_flags |= B_INVAL | B_NOCACHE;
3514 complete_jsegs(jseg);
3517 static inline struct jsegdep *
3519 struct inoref *inoref;
3521 struct jsegdep *jsegdep;
3523 jsegdep = inoref->if_jsegdep;
3524 inoref->if_jsegdep = NULL;
3530 * Called once a jremref has made it to stable store. The jremref is marked
3531 * complete and we attempt to free it. Any pagedeps writes sleeping waiting
3532 * for the jremref to complete will be awoken by free_jremref.
3535 handle_written_jremref(jremref)
3536 struct jremref *jremref;
3538 struct inodedep *inodedep;
3539 struct jsegdep *jsegdep;
3540 struct dirrem *dirrem;
3542 /* Grab the jsegdep. */
3543 jsegdep = inoref_jseg(&jremref->jr_ref);
3545 * Remove us from the inoref list.
3547 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino,
3549 panic("handle_written_jremref: Lost inodedep");
3550 TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps);
3552 * Complete the dirrem.
3554 dirrem = jremref->jr_dirrem;
3555 jremref->jr_dirrem = NULL;
3556 LIST_REMOVE(jremref, jr_deps);
3557 jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT;
3558 jwork_insert(&dirrem->dm_jwork, jsegdep);
3559 if (LIST_EMPTY(&dirrem->dm_jremrefhd) &&
3560 (dirrem->dm_state & COMPLETE) != 0)
3561 add_to_worklist(&dirrem->dm_list, 0);
3562 free_jremref(jremref);
3566 * Called once a jaddref has made it to stable store. The dependency is
3567 * marked complete and any dependent structures are added to the inode
3568 * bufwait list to be completed as soon as it is written. If a bitmap write
3569 * depends on this entry we move the inode into the inodedephd of the
3570 * bmsafemap dependency and attempt to remove the jaddref from the bmsafemap.
3573 handle_written_jaddref(jaddref)
3574 struct jaddref *jaddref;
3576 struct jsegdep *jsegdep;
3577 struct inodedep *inodedep;
3578 struct diradd *diradd;
3579 struct mkdir *mkdir;
3581 /* Grab the jsegdep. */
3582 jsegdep = inoref_jseg(&jaddref->ja_ref);
3585 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino,
3587 panic("handle_written_jaddref: Lost inodedep.");
3588 if (jaddref->ja_diradd == NULL)
3589 panic("handle_written_jaddref: No dependency");
3590 if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) {
3591 diradd = jaddref->ja_diradd;
3592 WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list);
3593 } else if (jaddref->ja_state & MKDIR_PARENT) {
3594 mkdir = jaddref->ja_mkdir;
3595 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list);
3596 } else if (jaddref->ja_state & MKDIR_BODY)
3597 mkdir = jaddref->ja_mkdir;
3599 panic("handle_written_jaddref: Unknown dependency %p",
3600 jaddref->ja_diradd);
3601 jaddref->ja_diradd = NULL; /* also clears ja_mkdir */
3603 * Remove us from the inode list.
3605 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps);
3607 * The mkdir may be waiting on the jaddref to clear before freeing.
3610 KASSERT(mkdir->md_list.wk_type == D_MKDIR,
3611 ("handle_written_jaddref: Incorrect type for mkdir %s",
3612 TYPENAME(mkdir->md_list.wk_type)));
3613 mkdir->md_jaddref = NULL;
3614 diradd = mkdir->md_diradd;
3615 mkdir->md_state |= DEPCOMPLETE;
3616 complete_mkdir(mkdir);
3618 jwork_insert(&diradd->da_jwork, jsegdep);
3619 if (jaddref->ja_state & NEWBLOCK) {
3620 inodedep->id_state |= ONDEPLIST;
3621 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd,
3624 free_jaddref(jaddref);
3628 * Called once a jnewblk journal is written. The allocdirect or allocindir
3629 * is placed in the bmsafemap to await notification of a written bitmap. If
3630 * the operation was canceled we add the segdep to the appropriate
3631 * dependency to free the journal space once the canceling operation
3635 handle_written_jnewblk(jnewblk)
3636 struct jnewblk *jnewblk;
3638 struct bmsafemap *bmsafemap;
3639 struct freefrag *freefrag;
3640 struct freework *freework;
3641 struct jsegdep *jsegdep;
3642 struct newblk *newblk;
3644 /* Grab the jsegdep. */
3645 jsegdep = jnewblk->jn_jsegdep;
3646 jnewblk->jn_jsegdep = NULL;
3647 if (jnewblk->jn_dep == NULL)
3648 panic("handle_written_jnewblk: No dependency for the segdep.");
3649 switch (jnewblk->jn_dep->wk_type) {
3654 * Add the written block to the bmsafemap so it can
3655 * be notified when the bitmap is on disk.
3657 newblk = WK_NEWBLK(jnewblk->jn_dep);
3658 newblk->nb_jnewblk = NULL;
3659 if ((newblk->nb_state & GOINGAWAY) == 0) {
3660 bmsafemap = newblk->nb_bmsafemap;
3661 newblk->nb_state |= ONDEPLIST;
3662 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk,
3665 jwork_insert(&newblk->nb_jwork, jsegdep);
3669 * A newblock being removed by a freefrag when replaced by
3672 freefrag = WK_FREEFRAG(jnewblk->jn_dep);
3673 freefrag->ff_jdep = NULL;
3674 jwork_insert(&freefrag->ff_jwork, jsegdep);
3678 * A direct block was removed by truncate.
3680 freework = WK_FREEWORK(jnewblk->jn_dep);
3681 freework->fw_jnewblk = NULL;
3682 jwork_insert(&freework->fw_freeblks->fb_jwork, jsegdep);
3685 panic("handle_written_jnewblk: Unknown type %d.",
3686 jnewblk->jn_dep->wk_type);
3688 jnewblk->jn_dep = NULL;
3689 free_jnewblk(jnewblk);
3693 * Cancel a jfreefrag that won't be needed, probably due to colliding with
3694 * an in-flight allocation that has not yet been committed. Divorce us
3695 * from the freefrag and mark it DEPCOMPLETE so that it may be added
3699 cancel_jfreefrag(jfreefrag)
3700 struct jfreefrag *jfreefrag;
3702 struct freefrag *freefrag;
3704 if (jfreefrag->fr_jsegdep) {
3705 free_jsegdep(jfreefrag->fr_jsegdep);
3706 jfreefrag->fr_jsegdep = NULL;
3708 freefrag = jfreefrag->fr_freefrag;
3709 jfreefrag->fr_freefrag = NULL;
3710 free_jfreefrag(jfreefrag);
3711 freefrag->ff_state |= DEPCOMPLETE;
3712 CTR1(KTR_SUJ, "cancel_jfreefrag: blkno %jd", freefrag->ff_blkno);
3716 * Free a jfreefrag when the parent freefrag is rendered obsolete.
3719 free_jfreefrag(jfreefrag)
3720 struct jfreefrag *jfreefrag;
3723 if (jfreefrag->fr_state & INPROGRESS)
3724 WORKLIST_REMOVE(&jfreefrag->fr_list);
3725 else if (jfreefrag->fr_state & ONWORKLIST)
3726 remove_from_journal(&jfreefrag->fr_list);
3727 if (jfreefrag->fr_freefrag != NULL)
3728 panic("free_jfreefrag: Still attached to a freefrag.");
3729 WORKITEM_FREE(jfreefrag, D_JFREEFRAG);
3733 * Called when the journal write for a jfreefrag completes. The parent
3734 * freefrag is added to the worklist if this completes its dependencies.
3737 handle_written_jfreefrag(jfreefrag)
3738 struct jfreefrag *jfreefrag;
3740 struct jsegdep *jsegdep;
3741 struct freefrag *freefrag;
3743 /* Grab the jsegdep. */
3744 jsegdep = jfreefrag->fr_jsegdep;
3745 jfreefrag->fr_jsegdep = NULL;
3746 freefrag = jfreefrag->fr_freefrag;
3747 if (freefrag == NULL)
3748 panic("handle_written_jfreefrag: No freefrag.");
3749 freefrag->ff_state |= DEPCOMPLETE;
3750 freefrag->ff_jdep = NULL;
3751 jwork_insert(&freefrag->ff_jwork, jsegdep);
3752 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE)
3753 add_to_worklist(&freefrag->ff_list, 0);
3754 jfreefrag->fr_freefrag = NULL;
3755 free_jfreefrag(jfreefrag);
3759 * Called when the journal write for a jfreeblk completes. The jfreeblk
3760 * is removed from the freeblks list of pending journal writes and the
3761 * jsegdep is moved to the freeblks jwork to be completed when all blocks
3762 * have been reclaimed.
3765 handle_written_jblkdep(jblkdep)
3766 struct jblkdep *jblkdep;
3768 struct freeblks *freeblks;
3769 struct jsegdep *jsegdep;
3771 /* Grab the jsegdep. */
3772 jsegdep = jblkdep->jb_jsegdep;
3773 jblkdep->jb_jsegdep = NULL;
3774 freeblks = jblkdep->jb_freeblks;
3775 LIST_REMOVE(jblkdep, jb_deps);
3776 jwork_insert(&freeblks->fb_jwork, jsegdep);
3778 * If the freeblks is all journaled, we can add it to the worklist.
3780 if (LIST_EMPTY(&freeblks->fb_jblkdephd) &&
3781 (freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE)
3782 add_to_worklist(&freeblks->fb_list, WK_NODELAY);
3784 free_jblkdep(jblkdep);
3787 static struct jsegdep *
3788 newjsegdep(struct worklist *wk)
3790 struct jsegdep *jsegdep;
3792 jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS);
3793 workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp);
3794 jsegdep->jd_seg = NULL;
3799 static struct jmvref *
3800 newjmvref(dp, ino, oldoff, newoff)
3806 struct jmvref *jmvref;
3808 jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS);
3809 workitem_alloc(&jmvref->jm_list, D_JMVREF, UFSTOVFS(dp->i_ump));
3810 jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE;
3811 jmvref->jm_parent = dp->i_number;
3812 jmvref->jm_ino = ino;
3813 jmvref->jm_oldoff = oldoff;
3814 jmvref->jm_newoff = newoff;
3820 * Allocate a new jremref that tracks the removal of ip from dp with the
3821 * directory entry offset of diroff. Mark the entry as ATTACHED and
3822 * DEPCOMPLETE as we have all the information required for the journal write
3823 * and the directory has already been removed from the buffer. The caller
3824 * is responsible for linking the jremref into the pagedep and adding it
3825 * to the journal to write. The MKDIR_PARENT flag is set if we're doing
3826 * a DOTDOT addition so handle_workitem_remove() can properly assign
3827 * the jsegdep when we're done.
3829 static struct jremref *
3830 newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip,
3831 off_t diroff, nlink_t nlink)
3833 struct jremref *jremref;
3835 jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS);
3836 workitem_alloc(&jremref->jr_list, D_JREMREF, UFSTOVFS(dp->i_ump));
3837 jremref->jr_state = ATTACHED;
3838 newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff,
3840 jremref->jr_dirrem = dirrem;
3846 newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff,
3847 nlink_t nlink, uint16_t mode)
3850 inoref->if_jsegdep = newjsegdep(&inoref->if_list);
3851 inoref->if_diroff = diroff;
3852 inoref->if_ino = ino;
3853 inoref->if_parent = parent;
3854 inoref->if_nlink = nlink;
3855 inoref->if_mode = mode;
3859 * Allocate a new jaddref to track the addition of ino to dp at diroff. The
3860 * directory offset may not be known until later. The caller is responsible
3861 * adding the entry to the journal when this information is available. nlink
3862 * should be the link count prior to the addition and mode is only required
3863 * to have the correct FMT.
3865 static struct jaddref *
3866 newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink,
3869 struct jaddref *jaddref;
3871 jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS);
3872 workitem_alloc(&jaddref->ja_list, D_JADDREF, UFSTOVFS(dp->i_ump));
3873 jaddref->ja_state = ATTACHED;
3874 jaddref->ja_mkdir = NULL;
3875 newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode);
3881 * Create a new free dependency for a freework. The caller is responsible
3882 * for adjusting the reference count when it has the lock held. The freedep
3883 * will track an outstanding bitmap write that will ultimately clear the
3884 * freework to continue.
3886 static struct freedep *
3887 newfreedep(struct freework *freework)
3889 struct freedep *freedep;
3891 freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS);
3892 workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp);
3893 freedep->fd_freework = freework;
3899 * Free a freedep structure once the buffer it is linked to is written. If
3900 * this is the last reference to the freework schedule it for completion.
3903 free_freedep(freedep)
3904 struct freedep *freedep;
3906 struct freework *freework;
3908 freework = freedep->fd_freework;
3909 freework->fw_freeblks->fb_cgwait--;
3910 if (--freework->fw_ref == 0)
3911 freework_enqueue(freework);
3912 WORKITEM_FREE(freedep, D_FREEDEP);
3916 * Allocate a new freework structure that may be a level in an indirect
3917 * when parent is not NULL or a top level block when it is. The top level
3918 * freework structures are allocated without lk held and before the freeblks
3919 * is visible outside of softdep_setup_freeblocks().
3921 static struct freework *
3922 newfreework(ump, freeblks, parent, lbn, nb, frags, off, journal)
3923 struct ufsmount *ump;
3924 struct freeblks *freeblks;
3925 struct freework *parent;
3932 struct freework *freework;
3934 freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS);
3935 workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp);
3936 freework->fw_state = ATTACHED;
3937 freework->fw_jnewblk = NULL;
3938 freework->fw_freeblks = freeblks;
3939 freework->fw_parent = parent;
3940 freework->fw_lbn = lbn;
3941 freework->fw_blkno = nb;
3942 freework->fw_frags = frags;
3943 freework->fw_indir = NULL;
3944 freework->fw_ref = (MOUNTEDSUJ(UFSTOVFS(ump)) == 0 || lbn >= -NXADDR)
3945 ? 0 : NINDIR(ump->um_fs) + 1;
3946 freework->fw_start = freework->fw_off = off;
3948 newjfreeblk(freeblks, lbn, nb, frags);
3949 if (parent == NULL) {
3951 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list);
3960 * Eliminate a jfreeblk for a block that does not need journaling.
3963 cancel_jfreeblk(freeblks, blkno)
3964 struct freeblks *freeblks;
3967 struct jfreeblk *jfreeblk;
3968 struct jblkdep *jblkdep;
3970 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) {
3971 if (jblkdep->jb_list.wk_type != D_JFREEBLK)
3973 jfreeblk = WK_JFREEBLK(&jblkdep->jb_list);
3974 if (jfreeblk->jf_blkno == blkno)
3977 if (jblkdep == NULL)
3979 CTR1(KTR_SUJ, "cancel_jfreeblk: blkno %jd", blkno);
3980 free_jsegdep(jblkdep->jb_jsegdep);
3981 LIST_REMOVE(jblkdep, jb_deps);
3982 WORKITEM_FREE(jfreeblk, D_JFREEBLK);
3986 * Allocate a new jfreeblk to journal top level block pointer when truncating
3987 * a file. The caller must add this to the worklist when lk is held.
3989 static struct jfreeblk *
3990 newjfreeblk(freeblks, lbn, blkno, frags)
3991 struct freeblks *freeblks;
3996 struct jfreeblk *jfreeblk;
3998 jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS);
3999 workitem_alloc(&jfreeblk->jf_dep.jb_list, D_JFREEBLK,
4000 freeblks->fb_list.wk_mp);
4001 jfreeblk->jf_dep.jb_jsegdep = newjsegdep(&jfreeblk->jf_dep.jb_list);
4002 jfreeblk->jf_dep.jb_freeblks = freeblks;
4003 jfreeblk->jf_ino = freeblks->fb_inum;
4004 jfreeblk->jf_lbn = lbn;
4005 jfreeblk->jf_blkno = blkno;
4006 jfreeblk->jf_frags = frags;
4007 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jfreeblk->jf_dep, jb_deps);
4013 * Allocate a new jtrunc to track a partial truncation.
4015 static struct jtrunc *
4016 newjtrunc(freeblks, size, extsize)
4017 struct freeblks *freeblks;
4021 struct jtrunc *jtrunc;
4023 jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS);
4024 workitem_alloc(&jtrunc->jt_dep.jb_list, D_JTRUNC,
4025 freeblks->fb_list.wk_mp);
4026 jtrunc->jt_dep.jb_jsegdep = newjsegdep(&jtrunc->jt_dep.jb_list);
4027 jtrunc->jt_dep.jb_freeblks = freeblks;
4028 jtrunc->jt_ino = freeblks->fb_inum;
4029 jtrunc->jt_size = size;
4030 jtrunc->jt_extsize = extsize;
4031 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jtrunc->jt_dep, jb_deps);
4037 * If we're canceling a new bitmap we have to search for another ref
4038 * to move into the bmsafemap dep. This might be better expressed
4039 * with another structure.
4042 move_newblock_dep(jaddref, inodedep)
4043 struct jaddref *jaddref;
4044 struct inodedep *inodedep;
4046 struct inoref *inoref;
4047 struct jaddref *jaddrefn;
4050 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref;
4051 inoref = TAILQ_NEXT(inoref, if_deps)) {
4052 if ((jaddref->ja_state & NEWBLOCK) &&
4053 inoref->if_list.wk_type == D_JADDREF) {
4054 jaddrefn = (struct jaddref *)inoref;
4058 if (jaddrefn == NULL)
4060 jaddrefn->ja_state &= ~(ATTACHED | UNDONE);
4061 jaddrefn->ja_state |= jaddref->ja_state &
4062 (ATTACHED | UNDONE | NEWBLOCK);
4063 jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK);
4064 jaddref->ja_state |= ATTACHED;
4065 LIST_REMOVE(jaddref, ja_bmdeps);
4066 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn,
4071 * Cancel a jaddref either before it has been written or while it is being
4072 * written. This happens when a link is removed before the add reaches
4073 * the disk. The jaddref dependency is kept linked into the bmsafemap
4074 * and inode to prevent the link count or bitmap from reaching the disk
4075 * until handle_workitem_remove() re-adjusts the counts and bitmaps as
4078 * Returns 1 if the canceled addref requires journaling of the remove and
4082 cancel_jaddref(jaddref, inodedep, wkhd)
4083 struct jaddref *jaddref;
4084 struct inodedep *inodedep;
4085 struct workhead *wkhd;
4087 struct inoref *inoref;
4088 struct jsegdep *jsegdep;
4091 KASSERT((jaddref->ja_state & COMPLETE) == 0,
4092 ("cancel_jaddref: Canceling complete jaddref"));
4093 if (jaddref->ja_state & (INPROGRESS | COMPLETE))
4097 if (inodedep == NULL)
4098 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino,
4100 panic("cancel_jaddref: Lost inodedep");
4102 * We must adjust the nlink of any reference operation that follows
4103 * us so that it is consistent with the in-memory reference. This
4104 * ensures that inode nlink rollbacks always have the correct link.
4107 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref;
4108 inoref = TAILQ_NEXT(inoref, if_deps)) {
4109 if (inoref->if_state & GOINGAWAY)
4114 jsegdep = inoref_jseg(&jaddref->ja_ref);
4115 if (jaddref->ja_state & NEWBLOCK)
4116 move_newblock_dep(jaddref, inodedep);
4117 wake_worklist(&jaddref->ja_list);
4118 jaddref->ja_mkdir = NULL;
4119 if (jaddref->ja_state & INPROGRESS) {
4120 jaddref->ja_state &= ~INPROGRESS;
4121 WORKLIST_REMOVE(&jaddref->ja_list);
4122 jwork_insert(wkhd, jsegdep);
4124 free_jsegdep(jsegdep);
4125 if (jaddref->ja_state & DEPCOMPLETE)
4126 remove_from_journal(&jaddref->ja_list);
4128 jaddref->ja_state |= (GOINGAWAY | DEPCOMPLETE);
4130 * Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove
4131 * can arrange for them to be freed with the bitmap. Otherwise we
4132 * no longer need this addref attached to the inoreflst and it
4133 * will incorrectly adjust nlink if we leave it.
4135 if ((jaddref->ja_state & NEWBLOCK) == 0) {
4136 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref,
4138 jaddref->ja_state |= COMPLETE;
4139 free_jaddref(jaddref);
4143 * Leave the head of the list for jsegdeps for fast merging.
4145 if (LIST_FIRST(wkhd) != NULL) {
4146 jaddref->ja_state |= ONWORKLIST;
4147 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list);
4149 WORKLIST_INSERT(wkhd, &jaddref->ja_list);
4155 * Attempt to free a jaddref structure when some work completes. This
4156 * should only succeed once the entry is written and all dependencies have
4160 free_jaddref(jaddref)
4161 struct jaddref *jaddref;
4164 if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE)
4166 if (jaddref->ja_ref.if_jsegdep)
4167 panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n",
4168 jaddref, jaddref->ja_state);
4169 if (jaddref->ja_state & NEWBLOCK)
4170 LIST_REMOVE(jaddref, ja_bmdeps);
4171 if (jaddref->ja_state & (INPROGRESS | ONWORKLIST))
4172 panic("free_jaddref: Bad state %p(0x%X)",
4173 jaddref, jaddref->ja_state);
4174 if (jaddref->ja_mkdir != NULL)
4175 panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state);
4176 WORKITEM_FREE(jaddref, D_JADDREF);
4180 * Free a jremref structure once it has been written or discarded.
4183 free_jremref(jremref)
4184 struct jremref *jremref;
4187 if (jremref->jr_ref.if_jsegdep)
4188 free_jsegdep(jremref->jr_ref.if_jsegdep);
4189 if (jremref->jr_state & INPROGRESS)
4190 panic("free_jremref: IO still pending");
4191 WORKITEM_FREE(jremref, D_JREMREF);
4195 * Free a jnewblk structure.
4198 free_jnewblk(jnewblk)
4199 struct jnewblk *jnewblk;
4202 if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE)
4204 LIST_REMOVE(jnewblk, jn_deps);
4205 if (jnewblk->jn_dep != NULL)
4206 panic("free_jnewblk: Dependency still attached.");
4207 WORKITEM_FREE(jnewblk, D_JNEWBLK);
4211 * Cancel a jnewblk which has been been made redundant by frag extension.
4214 cancel_jnewblk(jnewblk, wkhd)
4215 struct jnewblk *jnewblk;
4216 struct workhead *wkhd;
4218 struct jsegdep *jsegdep;
4220 CTR1(KTR_SUJ, "cancel_jnewblk: blkno %jd", jnewblk->jn_blkno);
4221 jsegdep = jnewblk->jn_jsegdep;
4222 if (jnewblk->jn_jsegdep == NULL || jnewblk->jn_dep == NULL)
4223 panic("cancel_jnewblk: Invalid state");
4224 jnewblk->jn_jsegdep = NULL;
4225 jnewblk->jn_dep = NULL;
4226 jnewblk->jn_state |= GOINGAWAY;
4227 if (jnewblk->jn_state & INPROGRESS) {
4228 jnewblk->jn_state &= ~INPROGRESS;
4229 WORKLIST_REMOVE(&jnewblk->jn_list);
4230 jwork_insert(wkhd, jsegdep);
4232 free_jsegdep(jsegdep);
4233 remove_from_journal(&jnewblk->jn_list);
4235 wake_worklist(&jnewblk->jn_list);
4236 WORKLIST_INSERT(wkhd, &jnewblk->jn_list);
4240 free_jblkdep(jblkdep)
4241 struct jblkdep *jblkdep;
4244 if (jblkdep->jb_list.wk_type == D_JFREEBLK)
4245 WORKITEM_FREE(jblkdep, D_JFREEBLK);
4246 else if (jblkdep->jb_list.wk_type == D_JTRUNC)
4247 WORKITEM_FREE(jblkdep, D_JTRUNC);
4249 panic("free_jblkdep: Unexpected type %s",
4250 TYPENAME(jblkdep->jb_list.wk_type));
4254 * Free a single jseg once it is no longer referenced in memory or on
4255 * disk. Reclaim journal blocks and dependencies waiting for the segment
4259 free_jseg(jseg, jblocks)
4261 struct jblocks *jblocks;
4263 struct freework *freework;
4266 * Free freework structures that were lingering to indicate freed
4267 * indirect blocks that forced journal write ordering on reallocate.
4269 while ((freework = LIST_FIRST(&jseg->js_indirs)) != NULL)
4270 indirblk_remove(freework);
4271 if (jblocks->jb_oldestseg == jseg)
4272 jblocks->jb_oldestseg = TAILQ_NEXT(jseg, js_next);
4273 TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next);
4274 jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size);
4275 KASSERT(LIST_EMPTY(&jseg->js_entries),
4276 ("free_jseg: Freed jseg has valid entries."));
4277 WORKITEM_FREE(jseg, D_JSEG);
4281 * Free all jsegs that meet the criteria for being reclaimed and update
4286 struct jblocks *jblocks;
4291 * Free only those jsegs which have none allocated before them to
4292 * preserve the journal space ordering.
4294 while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) {
4296 * Only reclaim space when nothing depends on this journal
4297 * set and another set has written that it is no longer
4300 if (jseg->js_refs != 0) {
4301 jblocks->jb_oldestseg = jseg;
4304 if ((jseg->js_state & ALLCOMPLETE) != ALLCOMPLETE)
4306 if (jseg->js_seq > jblocks->jb_oldestwrseq)
4309 * We can free jsegs that didn't write entries when
4310 * oldestwrseq == js_seq.
4312 if (jseg->js_seq == jblocks->jb_oldestwrseq &&
4315 free_jseg(jseg, jblocks);
4318 * If we exited the loop above we still must discover the
4319 * oldest valid segment.
4322 for (jseg = jblocks->jb_oldestseg; jseg != NULL;
4323 jseg = TAILQ_NEXT(jseg, js_next))
4324 if (jseg->js_refs != 0)
4326 jblocks->jb_oldestseg = jseg;
4328 * The journal has no valid records but some jsegs may still be
4329 * waiting on oldestwrseq to advance. We force a small record
4330 * out to permit these lingering records to be reclaimed.
4332 if (jblocks->jb_oldestseg == NULL && !TAILQ_EMPTY(&jblocks->jb_segs))
4333 jblocks->jb_needseg = 1;
4337 * Release one reference to a jseg and free it if the count reaches 0. This
4338 * should eventually reclaim journal space as well.
4345 KASSERT(jseg->js_refs > 0,
4346 ("free_jseg: Invalid refcnt %d", jseg->js_refs));
4347 if (--jseg->js_refs != 0)
4349 free_jsegs(jseg->js_jblocks);
4353 * Release a jsegdep and decrement the jseg count.
4356 free_jsegdep(jsegdep)
4357 struct jsegdep *jsegdep;
4360 if (jsegdep->jd_seg)
4361 rele_jseg(jsegdep->jd_seg);
4362 WORKITEM_FREE(jsegdep, D_JSEGDEP);
4366 * Wait for a journal item to make it to disk. Initiate journal processing
4371 struct worklist *wk;
4376 * Blocking journal waits cause slow synchronous behavior. Record
4377 * stats on the frequency of these blocking operations.
4379 if (waitfor == MNT_WAIT) {
4380 stat_journal_wait++;
4381 switch (wk->wk_type) {
4384 stat_jwait_filepage++;
4388 stat_jwait_freeblks++;
4391 stat_jwait_newblk++;
4401 * If IO has not started we process the journal. We can't mark the
4402 * worklist item as IOWAITING because we drop the lock while
4403 * processing the journal and the worklist entry may be freed after
4404 * this point. The caller may call back in and re-issue the request.
4406 if ((wk->wk_state & INPROGRESS) == 0) {
4407 softdep_process_journal(wk->wk_mp, wk, waitfor);
4408 if (waitfor != MNT_WAIT)
4412 if (waitfor != MNT_WAIT)
4414 wait_worklist(wk, "jwait");
4419 * Lookup an inodedep based on an inode pointer and set the nlinkdelta as
4420 * appropriate. This is a convenience function to reduce duplicate code
4421 * for the setup and revert functions below.
4423 static struct inodedep *
4424 inodedep_lookup_ip(ip)
4427 struct inodedep *inodedep;
4430 KASSERT(ip->i_nlink >= ip->i_effnlink,
4431 ("inodedep_lookup_ip: bad delta"));
4433 if (IS_SNAPSHOT(ip))
4435 (void) inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, dflags,
4437 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
4438 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked"));
4444 * Called prior to creating a new inode and linking it to a directory. The
4445 * jaddref structure must already be allocated by softdep_setup_inomapdep
4446 * and it is discovered here so we can initialize the mode and update
4450 softdep_setup_create(dp, ip)
4454 struct inodedep *inodedep;
4455 struct jaddref *jaddref;
4458 KASSERT(ip->i_nlink == 1,
4459 ("softdep_setup_create: Invalid link count."));
4462 inodedep = inodedep_lookup_ip(ip);
4463 if (DOINGSUJ(dvp)) {
4464 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4466 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
4467 ("softdep_setup_create: No addref structure present."));
4469 softdep_prelink(dvp, NULL);
4474 * Create a jaddref structure to track the addition of a DOTDOT link when
4475 * we are reparenting an inode as part of a rename. This jaddref will be
4476 * found by softdep_setup_directory_change. Adjusts nlinkdelta for
4477 * non-journaling softdep.
4480 softdep_setup_dotdot_link(dp, ip)
4484 struct inodedep *inodedep;
4485 struct jaddref *jaddref;
4493 * We don't set MKDIR_PARENT as this is not tied to a mkdir and
4494 * is used as a normal link would be.
4497 jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET,
4498 dp->i_effnlink - 1, dp->i_mode);
4500 inodedep = inodedep_lookup_ip(dp);
4502 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
4504 softdep_prelink(dvp, ITOV(ip));
4509 * Create a jaddref structure to track a new link to an inode. The directory
4510 * offset is not known until softdep_setup_directory_add or
4511 * softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling
4515 softdep_setup_link(dp, ip)
4519 struct inodedep *inodedep;
4520 struct jaddref *jaddref;
4526 jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1,
4529 inodedep = inodedep_lookup_ip(ip);
4531 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
4533 softdep_prelink(dvp, ITOV(ip));
4538 * Called to create the jaddref structures to track . and .. references as
4539 * well as lookup and further initialize the incomplete jaddref created
4540 * by softdep_setup_inomapdep when the inode was allocated. Adjusts
4541 * nlinkdelta for non-journaling softdep.
4544 softdep_setup_mkdir(dp, ip)
4548 struct inodedep *inodedep;
4549 struct jaddref *dotdotaddref;
4550 struct jaddref *dotaddref;
4551 struct jaddref *jaddref;
4555 dotaddref = dotdotaddref = NULL;
4556 if (DOINGSUJ(dvp)) {
4557 dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1,
4559 dotaddref->ja_state |= MKDIR_BODY;
4560 dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET,
4561 dp->i_effnlink - 1, dp->i_mode);
4562 dotdotaddref->ja_state |= MKDIR_PARENT;
4565 inodedep = inodedep_lookup_ip(ip);
4566 if (DOINGSUJ(dvp)) {
4567 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4569 KASSERT(jaddref != NULL,
4570 ("softdep_setup_mkdir: No addref structure present."));
4571 KASSERT(jaddref->ja_parent == dp->i_number,
4572 ("softdep_setup_mkdir: bad parent %d",
4573 jaddref->ja_parent));
4574 TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref,
4577 inodedep = inodedep_lookup_ip(dp);
4579 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst,
4580 &dotdotaddref->ja_ref, if_deps);
4581 softdep_prelink(ITOV(dp), NULL);
4586 * Called to track nlinkdelta of the inode and parent directories prior to
4587 * unlinking a directory.
4590 softdep_setup_rmdir(dp, ip)
4598 (void) inodedep_lookup_ip(ip);
4599 (void) inodedep_lookup_ip(dp);
4600 softdep_prelink(dvp, ITOV(ip));
4605 * Called to track nlinkdelta of the inode and parent directories prior to
4609 softdep_setup_unlink(dp, ip)
4617 (void) inodedep_lookup_ip(ip);
4618 (void) inodedep_lookup_ip(dp);
4619 softdep_prelink(dvp, ITOV(ip));
4624 * Called to release the journal structures created by a failed non-directory
4625 * creation. Adjusts nlinkdelta for non-journaling softdep.
4628 softdep_revert_create(dp, ip)
4632 struct inodedep *inodedep;
4633 struct jaddref *jaddref;
4638 inodedep = inodedep_lookup_ip(ip);
4639 if (DOINGSUJ(dvp)) {
4640 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4642 KASSERT(jaddref->ja_parent == dp->i_number,
4643 ("softdep_revert_create: addref parent mismatch"));
4644 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
4650 * Called to release the journal structures created by a failed dotdot link
4651 * creation. Adjusts nlinkdelta for non-journaling softdep.
4654 softdep_revert_dotdot_link(dp, ip)
4658 struct inodedep *inodedep;
4659 struct jaddref *jaddref;
4664 inodedep = inodedep_lookup_ip(dp);
4665 if (DOINGSUJ(dvp)) {
4666 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4668 KASSERT(jaddref->ja_parent == ip->i_number,
4669 ("softdep_revert_dotdot_link: addref parent mismatch"));
4670 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
4676 * Called to release the journal structures created by a failed link
4677 * addition. Adjusts nlinkdelta for non-journaling softdep.
4680 softdep_revert_link(dp, ip)
4684 struct inodedep *inodedep;
4685 struct jaddref *jaddref;
4690 inodedep = inodedep_lookup_ip(ip);
4691 if (DOINGSUJ(dvp)) {
4692 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4694 KASSERT(jaddref->ja_parent == dp->i_number,
4695 ("softdep_revert_link: addref parent mismatch"));
4696 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
4702 * Called to release the journal structures created by a failed mkdir
4703 * attempt. Adjusts nlinkdelta for non-journaling softdep.
4706 softdep_revert_mkdir(dp, ip)
4710 struct inodedep *inodedep;
4711 struct jaddref *jaddref;
4712 struct jaddref *dotaddref;
4718 inodedep = inodedep_lookup_ip(dp);
4719 if (DOINGSUJ(dvp)) {
4720 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4722 KASSERT(jaddref->ja_parent == ip->i_number,
4723 ("softdep_revert_mkdir: dotdot addref parent mismatch"));
4724 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
4726 inodedep = inodedep_lookup_ip(ip);
4727 if (DOINGSUJ(dvp)) {
4728 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
4730 KASSERT(jaddref->ja_parent == dp->i_number,
4731 ("softdep_revert_mkdir: addref parent mismatch"));
4732 dotaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref,
4733 inoreflst, if_deps);
4734 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
4735 KASSERT(dotaddref->ja_parent == ip->i_number,
4736 ("softdep_revert_mkdir: dot addref parent mismatch"));
4737 cancel_jaddref(dotaddref, inodedep, &inodedep->id_inowait);
4743 * Called to correct nlinkdelta after a failed rmdir.
4746 softdep_revert_rmdir(dp, ip)
4752 (void) inodedep_lookup_ip(ip);
4753 (void) inodedep_lookup_ip(dp);
4758 * Protecting the freemaps (or bitmaps).
4760 * To eliminate the need to execute fsck before mounting a filesystem
4761 * after a power failure, one must (conservatively) guarantee that the
4762 * on-disk copy of the bitmaps never indicate that a live inode or block is
4763 * free. So, when a block or inode is allocated, the bitmap should be
4764 * updated (on disk) before any new pointers. When a block or inode is
4765 * freed, the bitmap should not be updated until all pointers have been
4766 * reset. The latter dependency is handled by the delayed de-allocation
4767 * approach described below for block and inode de-allocation. The former
4768 * dependency is handled by calling the following procedure when a block or
4769 * inode is allocated. When an inode is allocated an "inodedep" is created
4770 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk.
4771 * Each "inodedep" is also inserted into the hash indexing structure so
4772 * that any additional link additions can be made dependent on the inode
4775 * The ufs filesystem maintains a number of free block counts (e.g., per
4776 * cylinder group, per cylinder and per <cylinder, rotational position> pair)
4777 * in addition to the bitmaps. These counts are used to improve efficiency
4778 * during allocation and therefore must be consistent with the bitmaps.
4779 * There is no convenient way to guarantee post-crash consistency of these
4780 * counts with simple update ordering, for two main reasons: (1) The counts
4781 * and bitmaps for a single cylinder group block are not in the same disk
4782 * sector. If a disk write is interrupted (e.g., by power failure), one may
4783 * be written and the other not. (2) Some of the counts are located in the
4784 * superblock rather than the cylinder group block. So, we focus our soft
4785 * updates implementation on protecting the bitmaps. When mounting a
4786 * filesystem, we recompute the auxiliary counts from the bitmaps.
4790 * Called just after updating the cylinder group block to allocate an inode.
4793 softdep_setup_inomapdep(bp, ip, newinum, mode)
4794 struct buf *bp; /* buffer for cylgroup block with inode map */
4795 struct inode *ip; /* inode related to allocation */
4796 ino_t newinum; /* new inode number being allocated */
4799 struct inodedep *inodedep;
4800 struct bmsafemap *bmsafemap;
4801 struct jaddref *jaddref;
4805 mp = UFSTOVFS(ip->i_ump);
4806 fs = ip->i_ump->um_fs;
4810 * Allocate the journal reference add structure so that the bitmap
4811 * can be dependent on it.
4813 if (MOUNTEDSUJ(mp)) {
4814 jaddref = newjaddref(ip, newinum, 0, 0, mode);
4815 jaddref->ja_state |= NEWBLOCK;
4819 * Create a dependency for the newly allocated inode.
4820 * Panic if it already exists as something is seriously wrong.
4821 * Otherwise add it to the dependency list for the buffer holding
4822 * the cylinder group map from which it was allocated.
4824 * We have to preallocate a bmsafemap entry in case it is needed
4825 * in bmsafemap_lookup since once we allocate the inodedep, we
4826 * have to finish initializing it before we can FREE_LOCK().
4827 * By preallocating, we avoid FREE_LOCK() while doing a malloc
4828 * in bmsafemap_lookup. We cannot call bmsafemap_lookup before
4829 * creating the inodedep as it can be freed during the time
4830 * that we FREE_LOCK() while allocating the inodedep. We must
4831 * call workitem_alloc() before entering the locked section as
4832 * it also acquires the lock and we must avoid trying doing so
4835 bmsafemap = malloc(sizeof(struct bmsafemap),
4836 M_BMSAFEMAP, M_SOFTDEP_FLAGS);
4837 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp);
4839 if ((inodedep_lookup(mp, newinum, DEPALLOC | NODELAY, &inodedep)))
4840 panic("softdep_setup_inomapdep: dependency %p for new"
4841 "inode already exists", inodedep);
4842 bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum), bmsafemap);
4844 LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps);
4845 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
4848 inodedep->id_state |= ONDEPLIST;
4849 LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps);
4851 inodedep->id_bmsafemap = bmsafemap;
4852 inodedep->id_state &= ~DEPCOMPLETE;
4857 * Called just after updating the cylinder group block to
4858 * allocate block or fragment.
4861 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags)
4862 struct buf *bp; /* buffer for cylgroup block with block map */
4863 struct mount *mp; /* filesystem doing allocation */
4864 ufs2_daddr_t newblkno; /* number of newly allocated block */
4865 int frags; /* Number of fragments. */
4866 int oldfrags; /* Previous number of fragments for extend. */
4868 struct newblk *newblk;
4869 struct bmsafemap *bmsafemap;
4870 struct jnewblk *jnewblk;
4873 fs = VFSTOUFS(mp)->um_fs;
4876 * Create a dependency for the newly allocated block.
4877 * Add it to the dependency list for the buffer holding
4878 * the cylinder group map from which it was allocated.
4880 if (MOUNTEDSUJ(mp)) {
4881 jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS);
4882 workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp);
4883 jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list);
4884 jnewblk->jn_state = ATTACHED;
4885 jnewblk->jn_blkno = newblkno;
4886 jnewblk->jn_frags = frags;
4887 jnewblk->jn_oldfrags = oldfrags;
4895 cgp = (struct cg *)bp->b_data;
4896 blksfree = cg_blksfree(cgp);
4897 bno = dtogd(fs, jnewblk->jn_blkno);
4898 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags;
4900 if (isset(blksfree, bno + i))
4901 panic("softdep_setup_blkmapdep: "
4902 "free fragment %d from %d-%d "
4903 "state 0x%X dep %p", i,
4904 jnewblk->jn_oldfrags,
4914 "softdep_setup_blkmapdep: blkno %jd frags %d oldfrags %d",
4915 newblkno, frags, oldfrags);
4917 if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0)
4918 panic("softdep_setup_blkmapdep: found block");
4919 newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp,
4920 dtog(fs, newblkno), NULL);
4922 jnewblk->jn_dep = (struct worklist *)newblk;
4923 LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps);
4925 newblk->nb_state |= ONDEPLIST;
4926 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps);
4928 newblk->nb_bmsafemap = bmsafemap;
4929 newblk->nb_jnewblk = jnewblk;
4933 #define BMSAFEMAP_HASH(fs, cg) \
4934 (&bmsafemap_hashtbl[((((register_t)(fs)) >> 13) + (cg)) & bmsafemap_hash])
4937 bmsafemap_find(bmsafemaphd, mp, cg, bmsafemapp)
4938 struct bmsafemap_hashhead *bmsafemaphd;
4941 struct bmsafemap **bmsafemapp;
4943 struct bmsafemap *bmsafemap;
4945 LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash)
4946 if (bmsafemap->sm_list.wk_mp == mp && bmsafemap->sm_cg == cg)
4949 *bmsafemapp = bmsafemap;
4958 * Find the bmsafemap associated with a cylinder group buffer.
4959 * If none exists, create one. The buffer must be locked when
4960 * this routine is called and this routine must be called with
4961 * the softdep lock held. To avoid giving up the lock while
4962 * allocating a new bmsafemap, a preallocated bmsafemap may be
4963 * provided. If it is provided but not needed, it is freed.
4965 static struct bmsafemap *
4966 bmsafemap_lookup(mp, bp, cg, newbmsafemap)
4970 struct bmsafemap *newbmsafemap;
4972 struct bmsafemap_hashhead *bmsafemaphd;
4973 struct bmsafemap *bmsafemap, *collision;
4974 struct worklist *wk;
4977 mtx_assert(&lk, MA_OWNED);
4979 LIST_FOREACH(wk, &bp->b_dep, wk_list)
4980 if (wk->wk_type == D_BMSAFEMAP) {
4982 WORKITEM_FREE(newbmsafemap,D_BMSAFEMAP);
4983 return (WK_BMSAFEMAP(wk));
4985 fs = VFSTOUFS(mp)->um_fs;
4986 bmsafemaphd = BMSAFEMAP_HASH(fs, cg);
4987 if (bmsafemap_find(bmsafemaphd, mp, cg, &bmsafemap) == 1) {
4989 WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP);
4993 bmsafemap = newbmsafemap;
4996 bmsafemap = malloc(sizeof(struct bmsafemap),
4997 M_BMSAFEMAP, M_SOFTDEP_FLAGS);
4998 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp);
5001 bmsafemap->sm_buf = bp;
5002 LIST_INIT(&bmsafemap->sm_inodedephd);
5003 LIST_INIT(&bmsafemap->sm_inodedepwr);
5004 LIST_INIT(&bmsafemap->sm_newblkhd);
5005 LIST_INIT(&bmsafemap->sm_newblkwr);
5006 LIST_INIT(&bmsafemap->sm_jaddrefhd);
5007 LIST_INIT(&bmsafemap->sm_jnewblkhd);
5008 LIST_INIT(&bmsafemap->sm_freehd);
5009 LIST_INIT(&bmsafemap->sm_freewr);
5010 if (bmsafemap_find(bmsafemaphd, mp, cg, &collision) == 1) {
5011 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP);
5014 bmsafemap->sm_cg = cg;
5015 LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash);
5016 LIST_INSERT_HEAD(&VFSTOUFS(mp)->softdep_dirtycg, bmsafemap, sm_next);
5017 WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list);
5022 * Direct block allocation dependencies.
5024 * When a new block is allocated, the corresponding disk locations must be
5025 * initialized (with zeros or new data) before the on-disk inode points to
5026 * them. Also, the freemap from which the block was allocated must be
5027 * updated (on disk) before the inode's pointer. These two dependencies are
5028 * independent of each other and are needed for all file blocks and indirect
5029 * blocks that are pointed to directly by the inode. Just before the
5030 * "in-core" version of the inode is updated with a newly allocated block
5031 * number, a procedure (below) is called to setup allocation dependency
5032 * structures. These structures are removed when the corresponding
5033 * dependencies are satisfied or when the block allocation becomes obsolete
5034 * (i.e., the file is deleted, the block is de-allocated, or the block is a
5035 * fragment that gets upgraded). All of these cases are handled in
5036 * procedures described later.
5038 * When a file extension causes a fragment to be upgraded, either to a larger
5039 * fragment or to a full block, the on-disk location may change (if the
5040 * previous fragment could not simply be extended). In this case, the old
5041 * fragment must be de-allocated, but not until after the inode's pointer has
5042 * been updated. In most cases, this is handled by later procedures, which
5043 * will construct a "freefrag" structure to be added to the workitem queue
5044 * when the inode update is complete (or obsolete). The main exception to
5045 * this is when an allocation occurs while a pending allocation dependency
5046 * (for the same block pointer) remains. This case is handled in the main
5047 * allocation dependency setup procedure by immediately freeing the
5048 * unreferenced fragments.
5051 softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp)
5052 struct inode *ip; /* inode to which block is being added */
5053 ufs_lbn_t off; /* block pointer within inode */
5054 ufs2_daddr_t newblkno; /* disk block number being added */
5055 ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */
5056 long newsize; /* size of new block */
5057 long oldsize; /* size of new block */
5058 struct buf *bp; /* bp for allocated block */
5060 struct allocdirect *adp, *oldadp;
5061 struct allocdirectlst *adphead;
5062 struct freefrag *freefrag;
5063 struct inodedep *inodedep;
5064 struct pagedep *pagedep;
5065 struct jnewblk *jnewblk;
5066 struct newblk *newblk;
5071 mp = UFSTOVFS(ip->i_ump);
5072 if (oldblkno && oldblkno != newblkno)
5073 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn);
5078 "softdep_setup_allocdirect: ino %d blkno %jd oldblkno %jd "
5079 "off %jd newsize %ld oldsize %d",
5080 ip->i_number, newblkno, oldblkno, off, newsize, oldsize);
5082 if (off >= NDADDR) {
5084 panic("softdep_setup_allocdirect: bad lbn %jd, off %jd",
5086 /* allocating an indirect block */
5088 panic("softdep_setup_allocdirect: non-zero indir");
5091 panic("softdep_setup_allocdirect: lbn %jd != off %jd",
5094 * Allocating a direct block.
5096 * If we are allocating a directory block, then we must
5097 * allocate an associated pagedep to track additions and
5100 if ((ip->i_mode & IFMT) == IFDIR)
5101 pagedep_lookup(mp, bp, ip->i_number, off, DEPALLOC,
5104 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0)
5105 panic("softdep_setup_allocdirect: lost block");
5106 KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
5107 ("softdep_setup_allocdirect: newblk already initialized"));
5109 * Convert the newblk to an allocdirect.
5111 newblk->nb_list.wk_type = D_ALLOCDIRECT;
5112 adp = (struct allocdirect *)newblk;
5113 newblk->nb_freefrag = freefrag;
5114 adp->ad_offset = off;
5115 adp->ad_oldblkno = oldblkno;
5116 adp->ad_newsize = newsize;
5117 adp->ad_oldsize = oldsize;
5120 * Finish initializing the journal.
5122 if ((jnewblk = newblk->nb_jnewblk) != NULL) {
5123 jnewblk->jn_ino = ip->i_number;
5124 jnewblk->jn_lbn = lbn;
5125 add_to_journal(&jnewblk->jn_list);
5127 if (freefrag && freefrag->ff_jdep != NULL &&
5128 freefrag->ff_jdep->wk_type == D_JFREEFRAG)
5129 add_to_journal(freefrag->ff_jdep);
5130 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep);
5131 adp->ad_inodedep = inodedep;
5133 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list);
5135 * The list of allocdirects must be kept in sorted and ascending
5136 * order so that the rollback routines can quickly determine the
5137 * first uncommitted block (the size of the file stored on disk
5138 * ends at the end of the lowest committed fragment, or if there
5139 * are no fragments, at the end of the highest committed block).
5140 * Since files generally grow, the typical case is that the new
5141 * block is to be added at the end of the list. We speed this
5142 * special case by checking against the last allocdirect in the
5143 * list before laboriously traversing the list looking for the
5146 adphead = &inodedep->id_newinoupdt;
5147 oldadp = TAILQ_LAST(adphead, allocdirectlst);
5148 if (oldadp == NULL || oldadp->ad_offset <= off) {
5149 /* insert at end of list */
5150 TAILQ_INSERT_TAIL(adphead, adp, ad_next);
5151 if (oldadp != NULL && oldadp->ad_offset == off)
5152 allocdirect_merge(adphead, adp, oldadp);
5156 TAILQ_FOREACH(oldadp, adphead, ad_next) {
5157 if (oldadp->ad_offset >= off)
5161 panic("softdep_setup_allocdirect: lost entry");
5162 /* insert in middle of list */
5163 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
5164 if (oldadp->ad_offset == off)
5165 allocdirect_merge(adphead, adp, oldadp);
5171 * Merge a newer and older journal record to be stored either in a
5172 * newblock or freefrag. This handles aggregating journal records for
5173 * fragment allocation into a second record as well as replacing a
5174 * journal free with an aborted journal allocation. A segment for the
5175 * oldest record will be placed on wkhd if it has been written. If not
5176 * the segment for the newer record will suffice.
5178 static struct worklist *
5179 jnewblk_merge(new, old, wkhd)
5180 struct worklist *new;
5181 struct worklist *old;
5182 struct workhead *wkhd;
5184 struct jnewblk *njnewblk;
5185 struct jnewblk *jnewblk;
5187 /* Handle NULLs to simplify callers. */
5192 /* Replace a jfreefrag with a jnewblk. */
5193 if (new->wk_type == D_JFREEFRAG) {
5194 if (WK_JNEWBLK(old)->jn_blkno != WK_JFREEFRAG(new)->fr_blkno)
5195 panic("jnewblk_merge: blkno mismatch: %p, %p",
5197 cancel_jfreefrag(WK_JFREEFRAG(new));
5200 if (old->wk_type != D_JNEWBLK || new->wk_type != D_JNEWBLK)
5201 panic("jnewblk_merge: Bad type: old %d new %d\n",
5202 old->wk_type, new->wk_type);
5204 * Handle merging of two jnewblk records that describe
5205 * different sets of fragments in the same block.
5207 jnewblk = WK_JNEWBLK(old);
5208 njnewblk = WK_JNEWBLK(new);
5209 if (jnewblk->jn_blkno != njnewblk->jn_blkno)
5210 panic("jnewblk_merge: Merging disparate blocks.");
5212 * The record may be rolled back in the cg.
5214 if (jnewblk->jn_state & UNDONE) {
5215 jnewblk->jn_state &= ~UNDONE;
5216 njnewblk->jn_state |= UNDONE;
5217 njnewblk->jn_state &= ~ATTACHED;
5220 * We modify the newer addref and free the older so that if neither
5221 * has been written the most up-to-date copy will be on disk. If
5222 * both have been written but rolled back we only temporarily need
5223 * one of them to fix the bits when the cg write completes.
5225 jnewblk->jn_state |= ATTACHED | COMPLETE;
5226 njnewblk->jn_oldfrags = jnewblk->jn_oldfrags;
5227 cancel_jnewblk(jnewblk, wkhd);
5228 WORKLIST_REMOVE(&jnewblk->jn_list);
5229 free_jnewblk(jnewblk);
5234 * Replace an old allocdirect dependency with a newer one.
5235 * This routine must be called with splbio interrupts blocked.
5238 allocdirect_merge(adphead, newadp, oldadp)
5239 struct allocdirectlst *adphead; /* head of list holding allocdirects */
5240 struct allocdirect *newadp; /* allocdirect being added */
5241 struct allocdirect *oldadp; /* existing allocdirect being checked */
5243 struct worklist *wk;
5244 struct freefrag *freefrag;
5247 mtx_assert(&lk, MA_OWNED);
5248 if (newadp->ad_oldblkno != oldadp->ad_newblkno ||
5249 newadp->ad_oldsize != oldadp->ad_newsize ||
5250 newadp->ad_offset >= NDADDR)
5251 panic("%s %jd != new %jd || old size %ld != new %ld",
5252 "allocdirect_merge: old blkno",
5253 (intmax_t)newadp->ad_oldblkno,
5254 (intmax_t)oldadp->ad_newblkno,
5255 newadp->ad_oldsize, oldadp->ad_newsize);
5256 newadp->ad_oldblkno = oldadp->ad_oldblkno;
5257 newadp->ad_oldsize = oldadp->ad_oldsize;
5259 * If the old dependency had a fragment to free or had never
5260 * previously had a block allocated, then the new dependency
5261 * can immediately post its freefrag and adopt the old freefrag.
5262 * This action is done by swapping the freefrag dependencies.
5263 * The new dependency gains the old one's freefrag, and the
5264 * old one gets the new one and then immediately puts it on
5265 * the worklist when it is freed by free_newblk. It is
5266 * not possible to do this swap when the old dependency had a
5267 * non-zero size but no previous fragment to free. This condition
5268 * arises when the new block is an extension of the old block.
5269 * Here, the first part of the fragment allocated to the new
5270 * dependency is part of the block currently claimed on disk by
5271 * the old dependency, so cannot legitimately be freed until the
5272 * conditions for the new dependency are fulfilled.
5274 freefrag = newadp->ad_freefrag;
5275 if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) {
5276 newadp->ad_freefrag = oldadp->ad_freefrag;
5277 oldadp->ad_freefrag = freefrag;
5280 * If we are tracking a new directory-block allocation,
5281 * move it from the old allocdirect to the new allocdirect.
5283 if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) {
5284 WORKLIST_REMOVE(wk);
5285 if (!LIST_EMPTY(&oldadp->ad_newdirblk))
5286 panic("allocdirect_merge: extra newdirblk");
5287 WORKLIST_INSERT(&newadp->ad_newdirblk, wk);
5289 TAILQ_REMOVE(adphead, oldadp, ad_next);
5291 * We need to move any journal dependencies over to the freefrag
5292 * that releases this block if it exists. Otherwise we are
5293 * extending an existing block and we'll wait until that is
5294 * complete to release the journal space and extend the
5295 * new journal to cover this old space as well.
5297 if (freefrag == NULL) {
5298 if (oldadp->ad_newblkno != newadp->ad_newblkno)
5299 panic("allocdirect_merge: %jd != %jd",
5300 oldadp->ad_newblkno, newadp->ad_newblkno);
5301 newadp->ad_block.nb_jnewblk = (struct jnewblk *)
5302 jnewblk_merge(&newadp->ad_block.nb_jnewblk->jn_list,
5303 &oldadp->ad_block.nb_jnewblk->jn_list,
5304 &newadp->ad_block.nb_jwork);
5305 oldadp->ad_block.nb_jnewblk = NULL;
5306 cancel_newblk(&oldadp->ad_block, NULL,
5307 &newadp->ad_block.nb_jwork);
5309 wk = (struct worklist *) cancel_newblk(&oldadp->ad_block,
5310 &freefrag->ff_list, &freefrag->ff_jwork);
5311 freefrag->ff_jdep = jnewblk_merge(freefrag->ff_jdep, wk,
5312 &freefrag->ff_jwork);
5314 free_newblk(&oldadp->ad_block);
5318 * Allocate a jfreefrag structure to journal a single block free.
5320 static struct jfreefrag *
5321 newjfreefrag(freefrag, ip, blkno, size, lbn)
5322 struct freefrag *freefrag;
5328 struct jfreefrag *jfreefrag;
5332 jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG,
5334 workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, UFSTOVFS(ip->i_ump));
5335 jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list);
5336 jfreefrag->fr_state = ATTACHED | DEPCOMPLETE;
5337 jfreefrag->fr_ino = ip->i_number;
5338 jfreefrag->fr_lbn = lbn;
5339 jfreefrag->fr_blkno = blkno;
5340 jfreefrag->fr_frags = numfrags(fs, size);
5341 jfreefrag->fr_freefrag = freefrag;
5347 * Allocate a new freefrag structure.
5349 static struct freefrag *
5350 newfreefrag(ip, blkno, size, lbn)
5356 struct freefrag *freefrag;
5359 CTR4(KTR_SUJ, "newfreefrag: ino %d blkno %jd size %ld lbn %jd",
5360 ip->i_number, blkno, size, lbn);
5362 if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag)
5363 panic("newfreefrag: frag size");
5364 freefrag = malloc(sizeof(struct freefrag),
5365 M_FREEFRAG, M_SOFTDEP_FLAGS);
5366 workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ip->i_ump));
5367 freefrag->ff_state = ATTACHED;
5368 LIST_INIT(&freefrag->ff_jwork);
5369 freefrag->ff_inum = ip->i_number;
5370 freefrag->ff_vtype = ITOV(ip)->v_type;
5371 freefrag->ff_blkno = blkno;
5372 freefrag->ff_fragsize = size;
5374 if (MOUNTEDSUJ(UFSTOVFS(ip->i_ump))) {
5375 freefrag->ff_jdep = (struct worklist *)
5376 newjfreefrag(freefrag, ip, blkno, size, lbn);
5378 freefrag->ff_state |= DEPCOMPLETE;
5379 freefrag->ff_jdep = NULL;
5386 * This workitem de-allocates fragments that were replaced during
5387 * file block allocation.
5390 handle_workitem_freefrag(freefrag)
5391 struct freefrag *freefrag;
5393 struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp);
5394 struct workhead wkhd;
5397 "handle_workitem_freefrag: ino %d blkno %jd size %ld",
5398 freefrag->ff_inum, freefrag->ff_blkno, freefrag->ff_fragsize);
5400 * It would be illegal to add new completion items to the
5401 * freefrag after it was schedule to be done so it must be
5402 * safe to modify the list head here.
5406 LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list);
5408 * If the journal has not been written we must cancel it here.
5410 if (freefrag->ff_jdep) {
5411 if (freefrag->ff_jdep->wk_type != D_JNEWBLK)
5412 panic("handle_workitem_freefrag: Unexpected type %d\n",
5413 freefrag->ff_jdep->wk_type);
5414 cancel_jnewblk(WK_JNEWBLK(freefrag->ff_jdep), &wkhd);
5417 ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno,
5418 freefrag->ff_fragsize, freefrag->ff_inum, freefrag->ff_vtype, &wkhd);
5420 WORKITEM_FREE(freefrag, D_FREEFRAG);
5425 * Set up a dependency structure for an external attributes data block.
5426 * This routine follows much of the structure of softdep_setup_allocdirect.
5427 * See the description of softdep_setup_allocdirect above for details.
5430 softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp)
5433 ufs2_daddr_t newblkno;
5434 ufs2_daddr_t oldblkno;
5439 struct allocdirect *adp, *oldadp;
5440 struct allocdirectlst *adphead;
5441 struct freefrag *freefrag;
5442 struct inodedep *inodedep;
5443 struct jnewblk *jnewblk;
5444 struct newblk *newblk;
5449 panic("softdep_setup_allocext: lbn %lld > NXADDR",
5453 mp = UFSTOVFS(ip->i_ump);
5454 if (oldblkno && oldblkno != newblkno)
5455 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn);
5460 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0)
5461 panic("softdep_setup_allocext: lost block");
5462 KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
5463 ("softdep_setup_allocext: newblk already initialized"));
5465 * Convert the newblk to an allocdirect.
5467 newblk->nb_list.wk_type = D_ALLOCDIRECT;
5468 adp = (struct allocdirect *)newblk;
5469 newblk->nb_freefrag = freefrag;
5470 adp->ad_offset = off;
5471 adp->ad_oldblkno = oldblkno;
5472 adp->ad_newsize = newsize;
5473 adp->ad_oldsize = oldsize;
5474 adp->ad_state |= EXTDATA;
5477 * Finish initializing the journal.
5479 if ((jnewblk = newblk->nb_jnewblk) != NULL) {
5480 jnewblk->jn_ino = ip->i_number;
5481 jnewblk->jn_lbn = lbn;
5482 add_to_journal(&jnewblk->jn_list);
5484 if (freefrag && freefrag->ff_jdep != NULL &&
5485 freefrag->ff_jdep->wk_type == D_JFREEFRAG)
5486 add_to_journal(freefrag->ff_jdep);
5487 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep);
5488 adp->ad_inodedep = inodedep;
5490 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list);
5492 * The list of allocdirects must be kept in sorted and ascending
5493 * order so that the rollback routines can quickly determine the
5494 * first uncommitted block (the size of the file stored on disk
5495 * ends at the end of the lowest committed fragment, or if there
5496 * are no fragments, at the end of the highest committed block).
5497 * Since files generally grow, the typical case is that the new
5498 * block is to be added at the end of the list. We speed this
5499 * special case by checking against the last allocdirect in the
5500 * list before laboriously traversing the list looking for the
5503 adphead = &inodedep->id_newextupdt;
5504 oldadp = TAILQ_LAST(adphead, allocdirectlst);
5505 if (oldadp == NULL || oldadp->ad_offset <= off) {
5506 /* insert at end of list */
5507 TAILQ_INSERT_TAIL(adphead, adp, ad_next);
5508 if (oldadp != NULL && oldadp->ad_offset == off)
5509 allocdirect_merge(adphead, adp, oldadp);
5513 TAILQ_FOREACH(oldadp, adphead, ad_next) {
5514 if (oldadp->ad_offset >= off)
5518 panic("softdep_setup_allocext: lost entry");
5519 /* insert in middle of list */
5520 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
5521 if (oldadp->ad_offset == off)
5522 allocdirect_merge(adphead, adp, oldadp);
5527 * Indirect block allocation dependencies.
5529 * The same dependencies that exist for a direct block also exist when
5530 * a new block is allocated and pointed to by an entry in a block of
5531 * indirect pointers. The undo/redo states described above are also
5532 * used here. Because an indirect block contains many pointers that
5533 * may have dependencies, a second copy of the entire in-memory indirect
5534 * block is kept. The buffer cache copy is always completely up-to-date.
5535 * The second copy, which is used only as a source for disk writes,
5536 * contains only the safe pointers (i.e., those that have no remaining
5537 * update dependencies). The second copy is freed when all pointers
5538 * are safe. The cache is not allowed to replace indirect blocks with
5539 * pending update dependencies. If a buffer containing an indirect
5540 * block with dependencies is written, these routines will mark it
5541 * dirty again. It can only be successfully written once all the
5542 * dependencies are removed. The ffs_fsync routine in conjunction with
5543 * softdep_sync_metadata work together to get all the dependencies
5544 * removed so that a file can be successfully written to disk. Three
5545 * procedures are used when setting up indirect block pointer
5546 * dependencies. The division is necessary because of the organization
5547 * of the "balloc" routine and because of the distinction between file
5548 * pages and file metadata blocks.
5552 * Allocate a new allocindir structure.
5554 static struct allocindir *
5555 newallocindir(ip, ptrno, newblkno, oldblkno, lbn)
5556 struct inode *ip; /* inode for file being extended */
5557 int ptrno; /* offset of pointer in indirect block */
5558 ufs2_daddr_t newblkno; /* disk block number being added */
5559 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */
5562 struct newblk *newblk;
5563 struct allocindir *aip;
5564 struct freefrag *freefrag;
5565 struct jnewblk *jnewblk;
5568 freefrag = newfreefrag(ip, oldblkno, ip->i_fs->fs_bsize, lbn);
5572 if (newblk_lookup(UFSTOVFS(ip->i_ump), newblkno, 0, &newblk) == 0)
5573 panic("new_allocindir: lost block");
5574 KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
5575 ("newallocindir: newblk already initialized"));
5576 newblk->nb_list.wk_type = D_ALLOCINDIR;
5577 newblk->nb_freefrag = freefrag;
5578 aip = (struct allocindir *)newblk;
5579 aip->ai_offset = ptrno;
5580 aip->ai_oldblkno = oldblkno;
5582 if ((jnewblk = newblk->nb_jnewblk) != NULL) {
5583 jnewblk->jn_ino = ip->i_number;
5584 jnewblk->jn_lbn = lbn;
5585 add_to_journal(&jnewblk->jn_list);
5587 if (freefrag && freefrag->ff_jdep != NULL &&
5588 freefrag->ff_jdep->wk_type == D_JFREEFRAG)
5589 add_to_journal(freefrag->ff_jdep);
5594 * Called just before setting an indirect block pointer
5595 * to a newly allocated file page.
5598 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
5599 struct inode *ip; /* inode for file being extended */
5600 ufs_lbn_t lbn; /* allocated block number within file */
5601 struct buf *bp; /* buffer with indirect blk referencing page */
5602 int ptrno; /* offset of pointer in indirect block */
5603 ufs2_daddr_t newblkno; /* disk block number being added */
5604 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */
5605 struct buf *nbp; /* buffer holding allocated page */
5607 struct inodedep *inodedep;
5608 struct freefrag *freefrag;
5609 struct allocindir *aip;
5610 struct pagedep *pagedep;
5614 if (lbn != nbp->b_lblkno)
5615 panic("softdep_setup_allocindir_page: lbn %jd != lblkno %jd",
5618 "softdep_setup_allocindir_page: ino %d blkno %jd oldblkno %jd "
5619 "lbn %jd", ip->i_number, newblkno, oldblkno, lbn);
5620 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page");
5621 mp = UFSTOVFS(ip->i_ump);
5622 aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn);
5624 if (IS_SNAPSHOT(ip))
5626 (void) inodedep_lookup(mp, ip->i_number, dflags, &inodedep);
5628 * If we are allocating a directory page, then we must
5629 * allocate an associated pagedep to track additions and
5632 if ((ip->i_mode & IFMT) == IFDIR)
5633 pagedep_lookup(mp, nbp, ip->i_number, lbn, DEPALLOC, &pagedep);
5634 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list);
5635 freefrag = setup_allocindir_phase2(bp, ip, inodedep, aip, lbn);
5638 handle_workitem_freefrag(freefrag);
5642 * Called just before setting an indirect block pointer to a
5643 * newly allocated indirect block.
5646 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
5647 struct buf *nbp; /* newly allocated indirect block */
5648 struct inode *ip; /* inode for file being extended */
5649 struct buf *bp; /* indirect block referencing allocated block */
5650 int ptrno; /* offset of pointer in indirect block */
5651 ufs2_daddr_t newblkno; /* disk block number being added */
5653 struct inodedep *inodedep;
5654 struct allocindir *aip;
5659 "softdep_setup_allocindir_meta: ino %d blkno %jd ptrno %d",
5660 ip->i_number, newblkno, ptrno);
5661 lbn = nbp->b_lblkno;
5662 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta");
5663 aip = newallocindir(ip, ptrno, newblkno, 0, lbn);
5665 if (IS_SNAPSHOT(ip))
5667 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, dflags, &inodedep);
5668 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list);
5669 if (setup_allocindir_phase2(bp, ip, inodedep, aip, lbn))
5670 panic("softdep_setup_allocindir_meta: Block already existed");
5675 indirdep_complete(indirdep)
5676 struct indirdep *indirdep;
5678 struct allocindir *aip;
5680 LIST_REMOVE(indirdep, ir_next);
5681 indirdep->ir_state |= DEPCOMPLETE;
5683 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) {
5684 LIST_REMOVE(aip, ai_next);
5685 free_newblk(&aip->ai_block);
5688 * If this indirdep is not attached to a buf it was simply waiting
5689 * on completion to clear completehd. free_indirdep() asserts
5690 * that nothing is dangling.
5692 if ((indirdep->ir_state & ONWORKLIST) == 0)
5693 free_indirdep(indirdep);
5696 static struct indirdep *
5697 indirdep_lookup(mp, ip, bp)
5702 struct indirdep *indirdep, *newindirdep;
5703 struct newblk *newblk;
5704 struct worklist *wk;
5708 mtx_assert(&lk, MA_OWNED);
5713 LIST_FOREACH(wk, &bp->b_dep, wk_list) {
5714 if (wk->wk_type != D_INDIRDEP)
5716 indirdep = WK_INDIRDEP(wk);
5719 /* Found on the buffer worklist, no new structure to free. */
5720 if (indirdep != NULL && newindirdep == NULL)
5722 if (indirdep != NULL && newindirdep != NULL)
5723 panic("indirdep_lookup: simultaneous create");
5724 /* None found on the buffer and a new structure is ready. */
5725 if (indirdep == NULL && newindirdep != NULL)
5727 /* None found and no new structure available. */
5729 newindirdep = malloc(sizeof(struct indirdep),
5730 M_INDIRDEP, M_SOFTDEP_FLAGS);
5731 workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp);
5732 newindirdep->ir_state = ATTACHED;
5733 if (ip->i_ump->um_fstype == UFS1)
5734 newindirdep->ir_state |= UFS1FMT;
5735 TAILQ_INIT(&newindirdep->ir_trunc);
5736 newindirdep->ir_saveddata = NULL;
5737 LIST_INIT(&newindirdep->ir_deplisthd);
5738 LIST_INIT(&newindirdep->ir_donehd);
5739 LIST_INIT(&newindirdep->ir_writehd);
5740 LIST_INIT(&newindirdep->ir_completehd);
5741 if (bp->b_blkno == bp->b_lblkno) {
5742 ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp,
5744 bp->b_blkno = blkno;
5746 newindirdep->ir_freeblks = NULL;
5747 newindirdep->ir_savebp =
5748 getblk(ip->i_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0);
5749 newindirdep->ir_bp = bp;
5750 BUF_KERNPROC(newindirdep->ir_savebp);
5751 bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount);
5754 indirdep = newindirdep;
5755 WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list);
5757 * If the block is not yet allocated we don't set DEPCOMPLETE so
5758 * that we don't free dependencies until the pointers are valid.
5759 * This could search b_dep for D_ALLOCDIRECT/D_ALLOCINDIR rather
5760 * than using the hash.
5762 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk))
5763 LIST_INSERT_HEAD(&newblk->nb_indirdeps, indirdep, ir_next);
5765 indirdep->ir_state |= DEPCOMPLETE;
5770 * Called to finish the allocation of the "aip" allocated
5771 * by one of the two routines above.
5773 static struct freefrag *
5774 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn)
5775 struct buf *bp; /* in-memory copy of the indirect block */
5776 struct inode *ip; /* inode for file being extended */
5777 struct inodedep *inodedep; /* Inodedep for ip */
5778 struct allocindir *aip; /* allocindir allocated by the above routines */
5779 ufs_lbn_t lbn; /* Logical block number for this block. */
5782 struct indirdep *indirdep;
5783 struct allocindir *oldaip;
5784 struct freefrag *freefrag;
5787 mtx_assert(&lk, MA_OWNED);
5788 mp = UFSTOVFS(ip->i_ump);
5790 if (bp->b_lblkno >= 0)
5791 panic("setup_allocindir_phase2: not indir blk");
5792 KASSERT(aip->ai_offset >= 0 && aip->ai_offset < NINDIR(fs),
5793 ("setup_allocindir_phase2: Bad offset %d", aip->ai_offset));
5794 indirdep = indirdep_lookup(mp, ip, bp);
5795 KASSERT(indirdep->ir_savebp != NULL,
5796 ("setup_allocindir_phase2 NULL ir_savebp"));
5797 aip->ai_indirdep = indirdep;
5799 * Check for an unwritten dependency for this indirect offset. If
5800 * there is, merge the old dependency into the new one. This happens
5801 * as a result of reallocblk only.
5804 if (aip->ai_oldblkno != 0) {
5805 LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, ai_next) {
5806 if (oldaip->ai_offset == aip->ai_offset) {
5807 freefrag = allocindir_merge(aip, oldaip);
5811 LIST_FOREACH(oldaip, &indirdep->ir_donehd, ai_next) {
5812 if (oldaip->ai_offset == aip->ai_offset) {
5813 freefrag = allocindir_merge(aip, oldaip);
5819 LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next);
5824 * Merge two allocindirs which refer to the same block. Move newblock
5825 * dependencies and setup the freefrags appropriately.
5827 static struct freefrag *
5828 allocindir_merge(aip, oldaip)
5829 struct allocindir *aip;
5830 struct allocindir *oldaip;
5832 struct freefrag *freefrag;
5833 struct worklist *wk;
5835 if (oldaip->ai_newblkno != aip->ai_oldblkno)
5836 panic("allocindir_merge: blkno");
5837 aip->ai_oldblkno = oldaip->ai_oldblkno;
5838 freefrag = aip->ai_freefrag;
5839 aip->ai_freefrag = oldaip->ai_freefrag;
5840 oldaip->ai_freefrag = NULL;
5841 KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag"));
5843 * If we are tracking a new directory-block allocation,
5844 * move it from the old allocindir to the new allocindir.
5846 if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) {
5847 WORKLIST_REMOVE(wk);
5848 if (!LIST_EMPTY(&oldaip->ai_newdirblk))
5849 panic("allocindir_merge: extra newdirblk");
5850 WORKLIST_INSERT(&aip->ai_newdirblk, wk);
5853 * We can skip journaling for this freefrag and just complete
5854 * any pending journal work for the allocindir that is being
5855 * removed after the freefrag completes.
5857 if (freefrag->ff_jdep)
5858 cancel_jfreefrag(WK_JFREEFRAG(freefrag->ff_jdep));
5859 LIST_REMOVE(oldaip, ai_next);
5860 freefrag->ff_jdep = (struct worklist *)cancel_newblk(&oldaip->ai_block,
5861 &freefrag->ff_list, &freefrag->ff_jwork);
5862 free_newblk(&oldaip->ai_block);
5868 setup_freedirect(freeblks, ip, i, needj)
5869 struct freeblks *freeblks;
5877 blkno = DIP(ip, i_db[i]);
5880 DIP_SET(ip, i_db[i], 0);
5881 frags = sblksize(ip->i_fs, ip->i_size, i);
5882 frags = numfrags(ip->i_fs, frags);
5883 newfreework(ip->i_ump, freeblks, NULL, i, blkno, frags, 0, needj);
5887 setup_freeext(freeblks, ip, i, needj)
5888 struct freeblks *freeblks;
5896 blkno = ip->i_din2->di_extb[i];
5899 ip->i_din2->di_extb[i] = 0;
5900 frags = sblksize(ip->i_fs, ip->i_din2->di_extsize, i);
5901 frags = numfrags(ip->i_fs, frags);
5902 newfreework(ip->i_ump, freeblks, NULL, -1 - i, blkno, frags, 0, needj);
5906 setup_freeindir(freeblks, ip, i, lbn, needj)
5907 struct freeblks *freeblks;
5915 blkno = DIP(ip, i_ib[i]);
5918 DIP_SET(ip, i_ib[i], 0);
5919 newfreework(ip->i_ump, freeblks, NULL, lbn, blkno, ip->i_fs->fs_frag,
5923 static inline struct freeblks *
5928 struct freeblks *freeblks;
5930 freeblks = malloc(sizeof(struct freeblks),
5931 M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO);
5932 workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp);
5933 LIST_INIT(&freeblks->fb_jblkdephd);
5934 LIST_INIT(&freeblks->fb_jwork);
5935 freeblks->fb_ref = 0;
5936 freeblks->fb_cgwait = 0;
5937 freeblks->fb_state = ATTACHED;
5938 freeblks->fb_uid = ip->i_uid;
5939 freeblks->fb_inum = ip->i_number;
5940 freeblks->fb_vtype = ITOV(ip)->v_type;
5941 freeblks->fb_modrev = DIP(ip, i_modrev);
5942 freeblks->fb_devvp = ip->i_devvp;
5943 freeblks->fb_chkcnt = 0;
5944 freeblks->fb_len = 0;
5950 trunc_indirdep(indirdep, freeblks, bp, off)
5951 struct indirdep *indirdep;
5952 struct freeblks *freeblks;
5956 struct allocindir *aip, *aipn;
5959 * The first set of allocindirs won't be in savedbp.
5961 LIST_FOREACH_SAFE(aip, &indirdep->ir_deplisthd, ai_next, aipn)
5962 if (aip->ai_offset > off)
5963 cancel_allocindir(aip, bp, freeblks, 1);
5964 LIST_FOREACH_SAFE(aip, &indirdep->ir_donehd, ai_next, aipn)
5965 if (aip->ai_offset > off)
5966 cancel_allocindir(aip, bp, freeblks, 1);
5968 * These will exist in savedbp.
5970 LIST_FOREACH_SAFE(aip, &indirdep->ir_writehd, ai_next, aipn)
5971 if (aip->ai_offset > off)
5972 cancel_allocindir(aip, NULL, freeblks, 0);
5973 LIST_FOREACH_SAFE(aip, &indirdep->ir_completehd, ai_next, aipn)
5974 if (aip->ai_offset > off)
5975 cancel_allocindir(aip, NULL, freeblks, 0);
5979 * Follow the chain of indirects down to lastlbn creating a freework
5980 * structure for each. This will be used to start indir_trunc() at
5981 * the right offset and create the journal records for the parrtial
5982 * truncation. A second step will handle the truncated dependencies.
5985 setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno)
5986 struct freeblks *freeblks;
5992 struct indirdep *indirdep;
5993 struct indirdep *indirn;
5994 struct freework *freework;
5995 struct newblk *newblk;
6009 mp = freeblks->fb_list.wk_mp;
6010 bp = getblk(ITOV(ip), lbn, mp->mnt_stat.f_iosize, 0, 0, 0);
6011 if ((bp->b_flags & B_CACHE) == 0) {
6012 bp->b_blkno = blkptrtodb(VFSTOUFS(mp), blkno);
6013 bp->b_iocmd = BIO_READ;
6014 bp->b_flags &= ~B_INVAL;
6015 bp->b_ioflags &= ~BIO_ERROR;
6016 vfs_busy_pages(bp, 0);
6017 bp->b_iooffset = dbtob(bp->b_blkno);
6019 curthread->td_ru.ru_inblock++;
6020 error = bufwait(bp);
6026 level = lbn_level(lbn);
6027 lbnadd = lbn_offset(ip->i_fs, level);
6029 * Compute the offset of the last block we want to keep. Store
6030 * in the freework the first block we want to completely free.
6032 off = (lastlbn - -(lbn + level)) / lbnadd;
6033 if (off + 1 == NINDIR(ip->i_fs))
6035 freework = newfreework(ip->i_ump, freeblks, NULL, lbn, blkno, 0, off+1,
6038 * Link the freework into the indirdep. This will prevent any new
6039 * allocations from proceeding until we are finished with the
6040 * truncate and the block is written.
6043 indirdep = indirdep_lookup(mp, ip, bp);
6044 if (indirdep->ir_freeblks)
6045 panic("setup_trunc_indir: indirdep already truncated.");
6046 TAILQ_INSERT_TAIL(&indirdep->ir_trunc, freework, fw_next);
6047 freework->fw_indir = indirdep;
6049 * Cancel any allocindirs that will not make it to disk.
6050 * We have to do this for all copies of the indirdep that
6051 * live on this newblk.
6053 if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
6054 newblk_lookup(mp, dbtofsb(ip->i_fs, bp->b_blkno), 0, &newblk);
6055 LIST_FOREACH(indirn, &newblk->nb_indirdeps, ir_next)
6056 trunc_indirdep(indirn, freeblks, bp, off);
6058 trunc_indirdep(indirdep, freeblks, bp, off);
6061 * Creation is protected by the buf lock. The saveddata is only
6062 * needed if a full truncation follows a partial truncation but it
6063 * is difficult to allocate in that case so we fetch it anyway.
6065 if (indirdep->ir_saveddata == NULL)
6066 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP,
6069 /* Fetch the blkno of the child and the zero start offset. */
6070 if (ip->i_ump->um_fstype == UFS1) {
6071 blkno = ((ufs1_daddr_t *)bp->b_data)[off];
6072 start = (uint8_t *)&((ufs1_daddr_t *)bp->b_data)[off+1];
6074 blkno = ((ufs2_daddr_t *)bp->b_data)[off];
6075 start = (uint8_t *)&((ufs2_daddr_t *)bp->b_data)[off+1];
6078 /* Zero the truncated pointers. */
6079 end = bp->b_data + bp->b_bcount;
6080 bzero(start, end - start);
6086 lbn++; /* adjust level */
6087 lbn -= (off * lbnadd);
6088 return setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno);
6092 * Complete the partial truncation of an indirect block setup by
6093 * setup_trunc_indir(). This zeros the truncated pointers in the saved
6094 * copy and writes them to disk before the freeblks is allowed to complete.
6097 complete_trunc_indir(freework)
6098 struct freework *freework;
6100 struct freework *fwn;
6101 struct indirdep *indirdep;
6106 indirdep = freework->fw_indir;
6108 bp = indirdep->ir_bp;
6109 /* See if the block was discarded. */
6112 /* Inline part of getdirtybuf(). We dont want bremfree. */
6113 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) == 0)
6116 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, &lk) == 0)
6120 mtx_assert(&lk, MA_OWNED);
6121 freework->fw_state |= DEPCOMPLETE;
6122 TAILQ_REMOVE(&indirdep->ir_trunc, freework, fw_next);
6124 * Zero the pointers in the saved copy.
6126 if (indirdep->ir_state & UFS1FMT)
6127 start = sizeof(ufs1_daddr_t);
6129 start = sizeof(ufs2_daddr_t);
6130 start *= freework->fw_start;
6131 count = indirdep->ir_savebp->b_bcount - start;
6132 start += (uintptr_t)indirdep->ir_savebp->b_data;
6133 bzero((char *)start, count);
6135 * We need to start the next truncation in the list if it has not
6138 fwn = TAILQ_FIRST(&indirdep->ir_trunc);
6140 if (fwn->fw_freeblks == indirdep->ir_freeblks)
6141 TAILQ_REMOVE(&indirdep->ir_trunc, fwn, fw_next);
6142 if ((fwn->fw_state & ONWORKLIST) == 0)
6143 freework_enqueue(fwn);
6146 * If bp is NULL the block was fully truncated, restore
6147 * the saved block list otherwise free it if it is no
6150 if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
6152 bcopy(indirdep->ir_saveddata,
6153 indirdep->ir_savebp->b_data,
6154 indirdep->ir_savebp->b_bcount);
6155 free(indirdep->ir_saveddata, M_INDIRDEP);
6156 indirdep->ir_saveddata = NULL;
6159 * When bp is NULL there is a full truncation pending. We
6160 * must wait for this full truncation to be journaled before
6161 * we can release this freework because the disk pointers will
6162 * never be written as zero.
6165 if (LIST_EMPTY(&indirdep->ir_freeblks->fb_jblkdephd))
6166 handle_written_freework(freework);
6168 WORKLIST_INSERT(&indirdep->ir_freeblks->fb_freeworkhd,
6169 &freework->fw_list);
6171 /* Complete when the real copy is written. */
6172 WORKLIST_INSERT(&bp->b_dep, &freework->fw_list);
6178 * Calculate the number of blocks we are going to release where datablocks
6179 * is the current total and length is the new file size.
6182 blkcount(fs, datablocks, length)
6184 ufs2_daddr_t datablocks;
6187 off_t totblks, numblks;
6190 numblks = howmany(length, fs->fs_bsize);
6191 if (numblks <= NDADDR) {
6192 totblks = howmany(length, fs->fs_fsize);
6195 totblks = blkstofrags(fs, numblks);
6198 * Count all single, then double, then triple indirects required.
6199 * Subtracting one indirects worth of blocks for each pass
6200 * acknowledges one of each pointed to by the inode.
6203 totblks += blkstofrags(fs, howmany(numblks, NINDIR(fs)));
6204 numblks -= NINDIR(fs);
6207 numblks = howmany(numblks, NINDIR(fs));
6210 totblks = fsbtodb(fs, totblks);
6212 * Handle sparse files. We can't reclaim more blocks than the inode
6213 * references. We will correct it later in handle_complete_freeblks()
6214 * when we know the real count.
6216 if (totblks > datablocks)
6218 return (datablocks - totblks);
6222 * Handle freeblocks for journaled softupdate filesystems.
6224 * Contrary to normal softupdates, we must preserve the block pointers in
6225 * indirects until their subordinates are free. This is to avoid journaling
6226 * every block that is freed which may consume more space than the journal
6227 * itself. The recovery program will see the free block journals at the
6228 * base of the truncated area and traverse them to reclaim space. The
6229 * pointers in the inode may be cleared immediately after the journal
6230 * records are written because each direct and indirect pointer in the
6231 * inode is recorded in a journal. This permits full truncation to proceed
6232 * asynchronously. The write order is journal -> inode -> cgs -> indirects.
6234 * The algorithm is as follows:
6235 * 1) Traverse the in-memory state and create journal entries to release
6236 * the relevant blocks and full indirect trees.
6237 * 2) Traverse the indirect block chain adding partial truncation freework
6238 * records to indirects in the path to lastlbn. The freework will
6239 * prevent new allocation dependencies from being satisfied in this
6240 * indirect until the truncation completes.
6241 * 3) Read and lock the inode block, performing an update with the new size
6242 * and pointers. This prevents truncated data from becoming valid on
6243 * disk through step 4.
6244 * 4) Reap unsatisfied dependencies that are beyond the truncated area,
6245 * eliminate journal work for those records that do not require it.
6246 * 5) Schedule the journal records to be written followed by the inode block.
6247 * 6) Allocate any necessary frags for the end of file.
6248 * 7) Zero any partially truncated blocks.
6250 * From this truncation proceeds asynchronously using the freework and
6251 * indir_trunc machinery. The file will not be extended again into a
6252 * partially truncated indirect block until all work is completed but
6253 * the normal dependency mechanism ensures that it is rolled back/forward
6254 * as appropriate. Further truncation may occur without delay and is
6255 * serialized in indir_trunc().
6258 softdep_journal_freeblocks(ip, cred, length, flags)
6259 struct inode *ip; /* The inode whose length is to be reduced */
6261 off_t length; /* The new length for the file */
6262 int flags; /* IO_EXT and/or IO_NORMAL */
6264 struct freeblks *freeblks, *fbn;
6265 struct worklist *wk, *wkn;
6266 struct inodedep *inodedep;
6267 struct jblkdep *jblkdep;
6268 struct allocdirect *adp, *adpn;
6273 ufs2_daddr_t extblocks, datablocks;
6274 ufs_lbn_t tmpval, lbn, lastlbn;
6275 int frags, lastoff, iboff, allocblock, needj, dflags, error, i;
6278 mp = UFSTOVFS(ip->i_ump);
6286 freeblks = newfreeblks(mp, ip);
6289 * If we're truncating a removed file that will never be written
6290 * we don't need to journal the block frees. The canceled journals
6291 * for the allocations will suffice.
6294 if (IS_SNAPSHOT(ip))
6296 inodedep_lookup(mp, ip->i_number, dflags, &inodedep);
6297 if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED &&
6300 CTR3(KTR_SUJ, "softdep_journal_freeblks: ip %d length %ld needj %d",
6301 ip->i_number, length, needj);
6304 * Calculate the lbn that we are truncating to. This results in -1
6305 * if we're truncating the 0 bytes. So it is the last lbn we want
6306 * to keep, not the first lbn we want to truncate.
6308 lastlbn = lblkno(fs, length + fs->fs_bsize - 1) - 1;
6309 lastoff = blkoff(fs, length);
6311 * Compute frags we are keeping in lastlbn. 0 means all.
6313 if (lastlbn >= 0 && lastlbn < NDADDR) {
6314 frags = fragroundup(fs, lastoff);
6315 /* adp offset of last valid allocdirect. */
6317 } else if (lastlbn > 0)
6319 if (fs->fs_magic == FS_UFS2_MAGIC)
6320 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
6322 * Handle normal data blocks and indirects. This section saves
6323 * values used after the inode update to complete frag and indirect
6326 if ((flags & IO_NORMAL) != 0) {
6328 * Handle truncation of whole direct and indirect blocks.
6330 for (i = iboff + 1; i < NDADDR; i++)
6331 setup_freedirect(freeblks, ip, i, needj);
6332 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR;
6333 i++, lbn += tmpval, tmpval *= NINDIR(fs)) {
6334 /* Release a whole indirect tree. */
6335 if (lbn > lastlbn) {
6336 setup_freeindir(freeblks, ip, i, -lbn -i,
6342 * Traverse partially truncated indirect tree.
6344 if (lbn <= lastlbn && lbn + tmpval - 1 > lastlbn)
6345 setup_trunc_indir(freeblks, ip, -lbn - i,
6346 lastlbn, DIP(ip, i_ib[i]));
6349 * Handle partial truncation to a frag boundary.
6355 oldfrags = blksize(fs, ip, lastlbn);
6356 blkno = DIP(ip, i_db[lastlbn]);
6357 if (blkno && oldfrags != frags) {
6359 oldfrags = numfrags(ip->i_fs, oldfrags);
6360 blkno += numfrags(ip->i_fs, frags);
6361 newfreework(ip->i_ump, freeblks, NULL, lastlbn,
6362 blkno, oldfrags, 0, needj);
6363 } else if (blkno == 0)
6367 * Add a journal record for partial truncate if we are
6368 * handling indirect blocks. Non-indirects need no extra
6371 if (length != 0 && lastlbn >= NDADDR) {
6372 ip->i_flag |= IN_TRUNCATED;
6373 newjtrunc(freeblks, length, 0);
6375 ip->i_size = length;
6376 DIP_SET(ip, i_size, ip->i_size);
6377 datablocks = DIP(ip, i_blocks) - extblocks;
6379 datablocks = blkcount(ip->i_fs, datablocks, length);
6380 freeblks->fb_len = length;
6382 if ((flags & IO_EXT) != 0) {
6383 for (i = 0; i < NXADDR; i++)
6384 setup_freeext(freeblks, ip, i, needj);
6385 ip->i_din2->di_extsize = 0;
6386 datablocks += extblocks;
6389 /* Reference the quotas in case the block count is wrong in the end. */
6390 quotaref(vp, freeblks->fb_quota);
6391 (void) chkdq(ip, -datablocks, NOCRED, 0);
6393 freeblks->fb_chkcnt = -datablocks;
6394 UFS_LOCK(ip->i_ump);
6395 fs->fs_pendingblocks += datablocks;
6396 UFS_UNLOCK(ip->i_ump);
6397 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks);
6399 * Handle truncation of incomplete alloc direct dependencies. We
6400 * hold the inode block locked to prevent incomplete dependencies
6401 * from reaching the disk while we are eliminating those that
6402 * have been truncated. This is a partially inlined ffs_update().
6405 ip->i_flag &= ~(IN_LAZYACCESS | IN_LAZYMOD | IN_MODIFIED);
6406 error = bread(ip->i_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
6407 (int)fs->fs_bsize, cred, &bp);
6410 softdep_error("softdep_journal_freeblocks", error);
6413 if (bp->b_bufsize == fs->fs_bsize)
6414 bp->b_flags |= B_CLUSTEROK;
6415 softdep_update_inodeblock(ip, bp, 0);
6416 if (ip->i_ump->um_fstype == UFS1)
6417 *((struct ufs1_dinode *)bp->b_data +
6418 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1;
6420 *((struct ufs2_dinode *)bp->b_data +
6421 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2;
6423 (void) inodedep_lookup(mp, ip->i_number, dflags, &inodedep);
6424 if ((inodedep->id_state & IOSTARTED) != 0)
6425 panic("softdep_setup_freeblocks: inode busy");
6427 * Add the freeblks structure to the list of operations that
6428 * must await the zero'ed inode being written to disk. If we
6429 * still have a bitmap dependency (needj), then the inode
6430 * has never been written to disk, so we can process the
6431 * freeblks below once we have deleted the dependencies.
6434 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list);
6436 freeblks->fb_state |= COMPLETE;
6437 if ((flags & IO_NORMAL) != 0) {
6438 TAILQ_FOREACH_SAFE(adp, &inodedep->id_inoupdt, ad_next, adpn) {
6439 if (adp->ad_offset > iboff)
6440 cancel_allocdirect(&inodedep->id_inoupdt, adp,
6443 * Truncate the allocdirect. We could eliminate
6444 * or modify journal records as well.
6446 else if (adp->ad_offset == iboff && frags)
6447 adp->ad_newsize = frags;
6450 if ((flags & IO_EXT) != 0)
6451 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != 0)
6452 cancel_allocdirect(&inodedep->id_extupdt, adp,
6455 * Scan the bufwait list for newblock dependencies that will never
6458 LIST_FOREACH_SAFE(wk, &inodedep->id_bufwait, wk_list, wkn) {
6459 if (wk->wk_type != D_ALLOCDIRECT)
6461 adp = WK_ALLOCDIRECT(wk);
6462 if (((flags & IO_NORMAL) != 0 && (adp->ad_offset > iboff)) ||
6463 ((flags & IO_EXT) != 0 && (adp->ad_state & EXTDATA))) {
6464 cancel_jfreeblk(freeblks, adp->ad_newblkno);
6465 cancel_newblk(WK_NEWBLK(wk), NULL, &freeblks->fb_jwork);
6466 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk);
6472 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps)
6473 add_to_journal(&jblkdep->jb_list);
6477 * Truncate dependency structures beyond length.
6479 trunc_dependencies(ip, freeblks, lastlbn, frags, flags);
6481 * This is only set when we need to allocate a fragment because
6482 * none existed at the end of a frag-sized file. It handles only
6483 * allocating a new, zero filled block.
6486 ip->i_size = length - lastoff;
6487 DIP_SET(ip, i_size, ip->i_size);
6488 error = UFS_BALLOC(vp, length - 1, 1, cred, BA_CLRBUF, &bp);
6490 softdep_error("softdep_journal_freeblks", error);
6493 ip->i_size = length;
6494 DIP_SET(ip, i_size, length);
6495 ip->i_flag |= IN_CHANGE | IN_UPDATE;
6496 allocbuf(bp, frags);
6499 } else if (lastoff != 0 && vp->v_type != VDIR) {
6503 * Zero the end of a truncated frag or block.
6505 size = sblksize(fs, length, lastlbn);
6506 error = bread(vp, lastlbn, size, cred, &bp);
6508 softdep_error("softdep_journal_freeblks", error);
6511 bzero((char *)bp->b_data + lastoff, size - lastoff);
6516 inodedep_lookup(mp, ip->i_number, dflags, &inodedep);
6517 TAILQ_INSERT_TAIL(&inodedep->id_freeblklst, freeblks, fb_next);
6518 freeblks->fb_state |= DEPCOMPLETE | ONDEPLIST;
6520 * We zero earlier truncations so they don't erroneously
6523 if (freeblks->fb_len == 0 && (flags & IO_NORMAL) != 0)
6524 TAILQ_FOREACH(fbn, &inodedep->id_freeblklst, fb_next)
6526 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE &&
6527 LIST_EMPTY(&freeblks->fb_jblkdephd))
6528 freeblks->fb_state |= INPROGRESS;
6533 handle_workitem_freeblocks(freeblks, 0);
6534 trunc_pages(ip, length, extblocks, flags);
6539 * Flush a JOP_SYNC to the journal.
6542 softdep_journal_fsync(ip)
6545 struct jfsync *jfsync;
6547 if ((ip->i_flag & IN_TRUNCATED) == 0)
6549 ip->i_flag &= ~IN_TRUNCATED;
6550 jfsync = malloc(sizeof(*jfsync), M_JFSYNC, M_SOFTDEP_FLAGS | M_ZERO);
6551 workitem_alloc(&jfsync->jfs_list, D_JFSYNC, UFSTOVFS(ip->i_ump));
6552 jfsync->jfs_size = ip->i_size;
6553 jfsync->jfs_ino = ip->i_number;
6555 add_to_journal(&jfsync->jfs_list);
6556 jwait(&jfsync->jfs_list, MNT_WAIT);
6561 * Block de-allocation dependencies.
6563 * When blocks are de-allocated, the on-disk pointers must be nullified before
6564 * the blocks are made available for use by other files. (The true
6565 * requirement is that old pointers must be nullified before new on-disk
6566 * pointers are set. We chose this slightly more stringent requirement to
6567 * reduce complexity.) Our implementation handles this dependency by updating
6568 * the inode (or indirect block) appropriately but delaying the actual block
6569 * de-allocation (i.e., freemap and free space count manipulation) until
6570 * after the updated versions reach stable storage. After the disk is
6571 * updated, the blocks can be safely de-allocated whenever it is convenient.
6572 * This implementation handles only the common case of reducing a file's
6573 * length to zero. Other cases are handled by the conventional synchronous
6576 * The ffs implementation with which we worked double-checks
6577 * the state of the block pointers and file size as it reduces
6578 * a file's length. Some of this code is replicated here in our
6579 * soft updates implementation. The freeblks->fb_chkcnt field is
6580 * used to transfer a part of this information to the procedure
6581 * that eventually de-allocates the blocks.
6583 * This routine should be called from the routine that shortens
6584 * a file's length, before the inode's size or block pointers
6585 * are modified. It will save the block pointer information for
6586 * later release and zero the inode so that the calling routine
6590 softdep_setup_freeblocks(ip, length, flags)
6591 struct inode *ip; /* The inode whose length is to be reduced */
6592 off_t length; /* The new length for the file */
6593 int flags; /* IO_EXT and/or IO_NORMAL */
6595 struct ufs1_dinode *dp1;
6596 struct ufs2_dinode *dp2;
6597 struct freeblks *freeblks;
6598 struct inodedep *inodedep;
6599 struct allocdirect *adp;
6602 ufs2_daddr_t extblocks, datablocks;
6604 int i, delay, error, dflags;
6608 CTR2(KTR_SUJ, "softdep_setup_freeblks: ip %d length %ld",
6609 ip->i_number, length);
6611 mp = UFSTOVFS(ip->i_ump);
6613 panic("softdep_setup_freeblocks: non-zero length");
6614 freeblks = newfreeblks(mp, ip);
6617 if (fs->fs_magic == FS_UFS2_MAGIC)
6618 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
6619 if ((flags & IO_NORMAL) != 0) {
6620 for (i = 0; i < NDADDR; i++)
6621 setup_freedirect(freeblks, ip, i, 0);
6622 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR;
6623 i++, lbn += tmpval, tmpval *= NINDIR(fs))
6624 setup_freeindir(freeblks, ip, i, -lbn -i, 0);
6626 DIP_SET(ip, i_size, 0);
6627 datablocks = DIP(ip, i_blocks) - extblocks;
6629 if ((flags & IO_EXT) != 0) {
6630 for (i = 0; i < NXADDR; i++)
6631 setup_freeext(freeblks, ip, i, 0);
6632 ip->i_din2->di_extsize = 0;
6633 datablocks += extblocks;
6636 /* Reference the quotas in case the block count is wrong in the end. */
6637 quotaref(ITOV(ip), freeblks->fb_quota);
6638 (void) chkdq(ip, -datablocks, NOCRED, 0);
6640 freeblks->fb_chkcnt = -datablocks;
6641 UFS_LOCK(ip->i_ump);
6642 fs->fs_pendingblocks += datablocks;
6643 UFS_UNLOCK(ip->i_ump);
6644 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks);
6646 * Push the zero'ed inode to to its disk buffer so that we are free
6647 * to delete its dependencies below. Once the dependencies are gone
6648 * the buffer can be safely released.
6650 if ((error = bread(ip->i_devvp,
6651 fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
6652 (int)fs->fs_bsize, NOCRED, &bp)) != 0) {
6654 softdep_error("softdep_setup_freeblocks", error);
6656 if (ip->i_ump->um_fstype == UFS1) {
6657 dp1 = ((struct ufs1_dinode *)bp->b_data +
6658 ino_to_fsbo(fs, ip->i_number));
6659 ip->i_din1->di_freelink = dp1->di_freelink;
6662 dp2 = ((struct ufs2_dinode *)bp->b_data +
6663 ino_to_fsbo(fs, ip->i_number));
6664 ip->i_din2->di_freelink = dp2->di_freelink;
6668 * Find and eliminate any inode dependencies.
6672 if (IS_SNAPSHOT(ip))
6674 (void) inodedep_lookup(mp, ip->i_number, dflags, &inodedep);
6675 if ((inodedep->id_state & IOSTARTED) != 0)
6676 panic("softdep_setup_freeblocks: inode busy");
6678 * Add the freeblks structure to the list of operations that
6679 * must await the zero'ed inode being written to disk. If we
6680 * still have a bitmap dependency (delay == 0), then the inode
6681 * has never been written to disk, so we can process the
6682 * freeblks below once we have deleted the dependencies.
6684 delay = (inodedep->id_state & DEPCOMPLETE);
6686 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list);
6688 freeblks->fb_state |= COMPLETE;
6690 * Because the file length has been truncated to zero, any
6691 * pending block allocation dependency structures associated
6692 * with this inode are obsolete and can simply be de-allocated.
6693 * We must first merge the two dependency lists to get rid of
6694 * any duplicate freefrag structures, then purge the merged list.
6695 * If we still have a bitmap dependency, then the inode has never
6696 * been written to disk, so we can free any fragments without delay.
6698 if (flags & IO_NORMAL) {
6699 merge_inode_lists(&inodedep->id_newinoupdt,
6700 &inodedep->id_inoupdt);
6701 while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != 0)
6702 cancel_allocdirect(&inodedep->id_inoupdt, adp,
6705 if (flags & IO_EXT) {
6706 merge_inode_lists(&inodedep->id_newextupdt,
6707 &inodedep->id_extupdt);
6708 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != 0)
6709 cancel_allocdirect(&inodedep->id_extupdt, adp,
6714 trunc_dependencies(ip, freeblks, -1, 0, flags);
6716 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0)
6717 (void) free_inodedep(inodedep);
6718 freeblks->fb_state |= DEPCOMPLETE;
6720 * If the inode with zeroed block pointers is now on disk
6721 * we can start freeing blocks.
6723 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE)
6724 freeblks->fb_state |= INPROGRESS;
6729 handle_workitem_freeblocks(freeblks, 0);
6730 trunc_pages(ip, length, extblocks, flags);
6734 * Eliminate pages from the page cache that back parts of this inode and
6735 * adjust the vnode pager's idea of our size. This prevents stale data
6736 * from hanging around in the page cache.
6739 trunc_pages(ip, length, extblocks, flags)
6742 ufs2_daddr_t extblocks;
6752 extend = OFF_TO_IDX(lblktosize(fs, -extblocks));
6753 if ((flags & IO_EXT) != 0)
6754 vn_pages_remove(vp, extend, 0);
6755 if ((flags & IO_NORMAL) == 0)
6757 BO_LOCK(&vp->v_bufobj);
6759 BO_UNLOCK(&vp->v_bufobj);
6761 * The vnode pager eliminates file pages we eliminate indirects
6764 vnode_pager_setsize(vp, length);
6766 * Calculate the end based on the last indirect we want to keep. If
6767 * the block extends into indirects we can just use the negative of
6768 * its lbn. Doubles and triples exist at lower numbers so we must
6769 * be careful not to remove those, if they exist. double and triple
6770 * indirect lbns do not overlap with others so it is not important
6771 * to verify how many levels are required.
6773 lbn = lblkno(fs, length);
6774 if (lbn >= NDADDR) {
6775 /* Calculate the virtual lbn of the triple indirect. */
6776 lbn = -lbn - (NIADDR - 1);
6777 end = OFF_TO_IDX(lblktosize(fs, lbn));
6780 vn_pages_remove(vp, OFF_TO_IDX(OFF_MAX), end);
6784 * See if the buf bp is in the range eliminated by truncation.
6787 trunc_check_buf(bp, blkoffp, lastlbn, lastoff, flags)
6797 /* Only match ext/normal blocks as appropriate. */
6798 if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) ||
6799 ((flags & IO_NORMAL) == 0 && (bp->b_xflags & BX_ALTDATA) == 0))
6801 /* ALTDATA is always a full truncation. */
6802 if ((bp->b_xflags & BX_ALTDATA) != 0)
6804 /* -1 is full truncation. */
6808 * If this is a partial truncate we only want those
6809 * blocks and indirect blocks that cover the range
6814 lbn = -(lbn + lbn_level(lbn));
6817 /* Here we only truncate lblkno if it's partial. */
6818 if (lbn == lastlbn) {
6827 * Eliminate any dependencies that exist in memory beyond lblkno:off
6830 trunc_dependencies(ip, freeblks, lastlbn, lastoff, flags)
6832 struct freeblks *freeblks;
6844 * We must wait for any I/O in progress to finish so that
6845 * all potential buffers on the dirty list will be visible.
6846 * Once they are all there, walk the list and get rid of
6854 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
6855 bp->b_vflags &= ~BV_SCANNED;
6857 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
6858 if (bp->b_vflags & BV_SCANNED)
6860 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) {
6861 bp->b_vflags |= BV_SCANNED;
6864 if ((bp = getdirtybuf(bp, BO_MTX(bo), MNT_WAIT)) == NULL)
6867 if (deallocate_dependencies(bp, freeblks, blkoff))
6875 * Now do the work of vtruncbuf while also matching indirect blocks.
6877 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs)
6878 bp->b_vflags &= ~BV_SCANNED;
6880 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) {
6881 if (bp->b_vflags & BV_SCANNED)
6883 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) {
6884 bp->b_vflags |= BV_SCANNED;
6888 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
6889 BO_MTX(bo)) == ENOLCK) {
6893 bp->b_vflags |= BV_SCANNED;
6898 allocbuf(bp, blkoff);
6901 bp->b_flags |= B_INVAL | B_NOCACHE | B_RELBUF;
6912 cancel_pagedep(pagedep, freeblks, blkoff)
6913 struct pagedep *pagedep;
6914 struct freeblks *freeblks;
6917 struct jremref *jremref;
6918 struct jmvref *jmvref;
6919 struct dirrem *dirrem, *tmp;
6923 * Copy any directory remove dependencies to the list
6924 * to be processed after the freeblks proceeds. If
6925 * directory entry never made it to disk they
6926 * can be dumped directly onto the work list.
6928 LIST_FOREACH_SAFE(dirrem, &pagedep->pd_dirremhd, dm_next, tmp) {
6929 /* Skip this directory removal if it is intended to remain. */
6930 if (dirrem->dm_offset < blkoff)
6933 * If there are any dirrems we wait for the journal write
6934 * to complete and then restart the buf scan as the lock
6937 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) {
6938 jwait(&jremref->jr_list, MNT_WAIT);
6941 LIST_REMOVE(dirrem, dm_next);
6942 dirrem->dm_dirinum = pagedep->pd_ino;
6943 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &dirrem->dm_list);
6945 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) {
6946 jwait(&jmvref->jm_list, MNT_WAIT);
6950 * When we're partially truncating a pagedep we just want to flush
6951 * journal entries and return. There can not be any adds in the
6952 * truncated portion of the directory and newblk must remain if
6953 * part of the block remains.
6958 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist)
6959 if (dap->da_offset > blkoff)
6960 panic("cancel_pagedep: diradd %p off %d > %d",
6961 dap, dap->da_offset, blkoff);
6962 for (i = 0; i < DAHASHSZ; i++)
6963 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist)
6964 if (dap->da_offset > blkoff)
6965 panic("cancel_pagedep: diradd %p off %d > %d",
6966 dap, dap->da_offset, blkoff);
6970 * There should be no directory add dependencies present
6971 * as the directory could not be truncated until all
6972 * children were removed.
6974 KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL,
6975 ("deallocate_dependencies: pendinghd != NULL"));
6976 for (i = 0; i < DAHASHSZ; i++)
6977 KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL,
6978 ("deallocate_dependencies: diraddhd != NULL"));
6979 if ((pagedep->pd_state & NEWBLOCK) != 0)
6980 free_newdirblk(pagedep->pd_newdirblk);
6981 if (free_pagedep(pagedep) == 0)
6982 panic("Failed to free pagedep %p", pagedep);
6987 * Reclaim any dependency structures from a buffer that is about to
6988 * be reallocated to a new vnode. The buffer must be locked, thus,
6989 * no I/O completion operations can occur while we are manipulating
6990 * its associated dependencies. The mutex is held so that other I/O's
6991 * associated with related dependencies do not occur.
6994 deallocate_dependencies(bp, freeblks, off)
6996 struct freeblks *freeblks;
6999 struct indirdep *indirdep;
7000 struct pagedep *pagedep;
7001 struct allocdirect *adp;
7002 struct worklist *wk, *wkn;
7005 LIST_FOREACH_SAFE(wk, &bp->b_dep, wk_list, wkn) {
7006 switch (wk->wk_type) {
7008 indirdep = WK_INDIRDEP(wk);
7009 if (bp->b_lblkno >= 0 ||
7010 bp->b_blkno != indirdep->ir_savebp->b_lblkno)
7011 panic("deallocate_dependencies: not indir");
7012 cancel_indirdep(indirdep, bp, freeblks);
7016 pagedep = WK_PAGEDEP(wk);
7017 if (cancel_pagedep(pagedep, freeblks, off)) {
7025 * Simply remove the allocindir, we'll find it via
7026 * the indirdep where we can clear pointers if
7029 WORKLIST_REMOVE(wk);
7034 * A truncation is waiting for the zero'd pointers
7035 * to be written. It can be freed when the freeblks
7038 WORKLIST_REMOVE(wk);
7039 wk->wk_state |= ONDEPLIST;
7040 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk);
7044 adp = WK_ALLOCDIRECT(wk);
7049 panic("deallocate_dependencies: Unexpected type %s",
7050 TYPENAME(wk->wk_type));
7056 * Don't throw away this buf, we were partially truncating and
7057 * some deps may always remain.
7061 bp->b_vflags |= BV_SCANNED;
7064 bp->b_flags |= B_INVAL | B_NOCACHE;
7070 * An allocdirect is being canceled due to a truncate. We must make sure
7071 * the journal entry is released in concert with the blkfree that releases
7072 * the storage. Completed journal entries must not be released until the
7073 * space is no longer pointed to by the inode or in the bitmap.
7076 cancel_allocdirect(adphead, adp, freeblks)
7077 struct allocdirectlst *adphead;
7078 struct allocdirect *adp;
7079 struct freeblks *freeblks;
7081 struct freework *freework;
7082 struct newblk *newblk;
7083 struct worklist *wk;
7085 TAILQ_REMOVE(adphead, adp, ad_next);
7086 newblk = (struct newblk *)adp;
7089 * Find the correct freework structure.
7091 LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) {
7092 if (wk->wk_type != D_FREEWORK)
7094 freework = WK_FREEWORK(wk);
7095 if (freework->fw_blkno == newblk->nb_newblkno)
7098 if (freework == NULL)
7099 panic("cancel_allocdirect: Freework not found");
7101 * If a newblk exists at all we still have the journal entry that
7102 * initiated the allocation so we do not need to journal the free.
7104 cancel_jfreeblk(freeblks, freework->fw_blkno);
7106 * If the journal hasn't been written the jnewblk must be passed
7107 * to the call to ffs_blkfree that reclaims the space. We accomplish
7108 * this by linking the journal dependency into the freework to be
7109 * freed when freework_freeblock() is called. If the journal has
7110 * been written we can simply reclaim the journal space when the
7111 * freeblks work is complete.
7113 freework->fw_jnewblk = cancel_newblk(newblk, &freework->fw_list,
7114 &freeblks->fb_jwork);
7115 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list);
7120 * Cancel a new block allocation. May be an indirect or direct block. We
7121 * remove it from various lists and return any journal record that needs to
7122 * be resolved by the caller.
7124 * A special consideration is made for indirects which were never pointed
7125 * at on disk and will never be found once this block is released.
7127 static struct jnewblk *
7128 cancel_newblk(newblk, wk, wkhd)
7129 struct newblk *newblk;
7130 struct worklist *wk;
7131 struct workhead *wkhd;
7133 struct jnewblk *jnewblk;
7135 CTR1(KTR_SUJ, "cancel_newblk: blkno %jd", newblk->nb_newblkno);
7137 newblk->nb_state |= GOINGAWAY;
7139 * Previously we traversed the completedhd on each indirdep
7140 * attached to this newblk to cancel them and gather journal
7141 * work. Since we need only the oldest journal segment and
7142 * the lowest point on the tree will always have the oldest
7143 * journal segment we are free to release the segments
7144 * of any subordinates and may leave the indirdep list to
7145 * indirdep_complete() when this newblk is freed.
7147 if (newblk->nb_state & ONDEPLIST) {
7148 newblk->nb_state &= ~ONDEPLIST;
7149 LIST_REMOVE(newblk, nb_deps);
7151 if (newblk->nb_state & ONWORKLIST)
7152 WORKLIST_REMOVE(&newblk->nb_list);
7154 * If the journal entry hasn't been written we save a pointer to
7155 * the dependency that frees it until it is written or the
7156 * superseding operation completes.
7158 jnewblk = newblk->nb_jnewblk;
7159 if (jnewblk != NULL && wk != NULL) {
7160 newblk->nb_jnewblk = NULL;
7161 jnewblk->jn_dep = wk;
7163 if (!LIST_EMPTY(&newblk->nb_jwork))
7164 jwork_move(wkhd, &newblk->nb_jwork);
7166 * When truncating we must free the newdirblk early to remove
7167 * the pagedep from the hash before returning.
7169 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL)
7170 free_newdirblk(WK_NEWDIRBLK(wk));
7171 if (!LIST_EMPTY(&newblk->nb_newdirblk))
7172 panic("cancel_newblk: extra newdirblk");
7178 * Schedule the freefrag associated with a newblk to be released once
7179 * the pointers are written and the previous block is no longer needed.
7182 newblk_freefrag(newblk)
7183 struct newblk *newblk;
7185 struct freefrag *freefrag;
7187 if (newblk->nb_freefrag == NULL)
7189 freefrag = newblk->nb_freefrag;
7190 newblk->nb_freefrag = NULL;
7191 freefrag->ff_state |= COMPLETE;
7192 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE)
7193 add_to_worklist(&freefrag->ff_list, 0);
7197 * Free a newblk. Generate a new freefrag work request if appropriate.
7198 * This must be called after the inode pointer and any direct block pointers
7199 * are valid or fully removed via truncate or frag extension.
7203 struct newblk *newblk;
7205 struct indirdep *indirdep;
7206 struct worklist *wk;
7208 KASSERT(newblk->nb_jnewblk == NULL,
7209 ("free_newblk; jnewblk %p still attached", newblk->nb_jnewblk));
7210 mtx_assert(&lk, MA_OWNED);
7211 newblk_freefrag(newblk);
7212 if (newblk->nb_state & ONDEPLIST)
7213 LIST_REMOVE(newblk, nb_deps);
7214 if (newblk->nb_state & ONWORKLIST)
7215 WORKLIST_REMOVE(&newblk->nb_list);
7216 LIST_REMOVE(newblk, nb_hash);
7217 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL)
7218 free_newdirblk(WK_NEWDIRBLK(wk));
7219 if (!LIST_EMPTY(&newblk->nb_newdirblk))
7220 panic("free_newblk: extra newdirblk");
7221 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL)
7222 indirdep_complete(indirdep);
7223 handle_jwork(&newblk->nb_jwork);
7224 newblk->nb_list.wk_type = D_NEWBLK;
7225 WORKITEM_FREE(newblk, D_NEWBLK);
7229 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep.
7230 * This routine must be called with splbio interrupts blocked.
7233 free_newdirblk(newdirblk)
7234 struct newdirblk *newdirblk;
7236 struct pagedep *pagedep;
7238 struct worklist *wk;
7240 mtx_assert(&lk, MA_OWNED);
7241 WORKLIST_REMOVE(&newdirblk->db_list);
7243 * If the pagedep is still linked onto the directory buffer
7244 * dependency chain, then some of the entries on the
7245 * pd_pendinghd list may not be committed to disk yet. In
7246 * this case, we will simply clear the NEWBLOCK flag and
7247 * let the pd_pendinghd list be processed when the pagedep
7248 * is next written. If the pagedep is no longer on the buffer
7249 * dependency chain, then all the entries on the pd_pending
7250 * list are committed to disk and we can free them here.
7252 pagedep = newdirblk->db_pagedep;
7253 pagedep->pd_state &= ~NEWBLOCK;
7254 if ((pagedep->pd_state & ONWORKLIST) == 0) {
7255 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
7256 free_diradd(dap, NULL);
7258 * If no dependencies remain, the pagedep will be freed.
7260 free_pagedep(pagedep);
7262 /* Should only ever be one item in the list. */
7263 while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) {
7264 WORKLIST_REMOVE(wk);
7265 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY);
7267 WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
7271 * Prepare an inode to be freed. The actual free operation is not
7272 * done until the zero'ed inode has been written to disk.
7275 softdep_freefile(pvp, ino, mode)
7280 struct inode *ip = VTOI(pvp);
7281 struct inodedep *inodedep;
7282 struct freefile *freefile;
7283 struct freeblks *freeblks;
7286 * This sets up the inode de-allocation dependency.
7288 freefile = malloc(sizeof(struct freefile),
7289 M_FREEFILE, M_SOFTDEP_FLAGS);
7290 workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount);
7291 freefile->fx_mode = mode;
7292 freefile->fx_oldinum = ino;
7293 freefile->fx_devvp = ip->i_devvp;
7294 LIST_INIT(&freefile->fx_jwork);
7295 UFS_LOCK(ip->i_ump);
7296 ip->i_fs->fs_pendinginodes += 1;
7297 UFS_UNLOCK(ip->i_ump);
7300 * If the inodedep does not exist, then the zero'ed inode has
7301 * been written to disk. If the allocated inode has never been
7302 * written to disk, then the on-disk inode is zero'ed. In either
7303 * case we can free the file immediately. If the journal was
7304 * canceled before being written the inode will never make it to
7305 * disk and we must send the canceled journal entrys to
7306 * ffs_freefile() to be cleared in conjunction with the bitmap.
7307 * Any blocks waiting on the inode to write can be safely freed
7308 * here as it will never been written.
7311 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep);
7314 * Clear out freeblks that no longer need to reference
7318 TAILQ_FIRST(&inodedep->id_freeblklst)) != NULL) {
7319 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks,
7321 freeblks->fb_state &= ~ONDEPLIST;
7324 * Remove this inode from the unlinked list.
7326 if (inodedep->id_state & UNLINKED) {
7328 * Save the journal work to be freed with the bitmap
7329 * before we clear UNLINKED. Otherwise it can be lost
7330 * if the inode block is written.
7332 handle_bufwait(inodedep, &freefile->fx_jwork);
7333 clear_unlinked_inodedep(inodedep);
7334 /* Re-acquire inodedep as we've dropped lk. */
7335 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep);
7338 if (inodedep == NULL || check_inode_unwritten(inodedep)) {
7340 handle_workitem_freefile(freefile);
7343 if ((inodedep->id_state & DEPCOMPLETE) == 0)
7344 inodedep->id_state |= GOINGAWAY;
7345 WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list);
7347 if (ip->i_number == ino)
7348 ip->i_flag |= IN_MODIFIED;
7352 * Check to see if an inode has never been written to disk. If
7353 * so free the inodedep and return success, otherwise return failure.
7354 * This routine must be called with splbio interrupts blocked.
7356 * If we still have a bitmap dependency, then the inode has never
7357 * been written to disk. Drop the dependency as it is no longer
7358 * necessary since the inode is being deallocated. We set the
7359 * ALLCOMPLETE flags since the bitmap now properly shows that the
7360 * inode is not allocated. Even if the inode is actively being
7361 * written, it has been rolled back to its zero'ed state, so we
7362 * are ensured that a zero inode is what is on the disk. For short
7363 * lived files, this change will usually result in removing all the
7364 * dependencies from the inode so that it can be freed immediately.
7367 check_inode_unwritten(inodedep)
7368 struct inodedep *inodedep;
7371 mtx_assert(&lk, MA_OWNED);
7373 if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 ||
7374 !LIST_EMPTY(&inodedep->id_dirremhd) ||
7375 !LIST_EMPTY(&inodedep->id_pendinghd) ||
7376 !LIST_EMPTY(&inodedep->id_bufwait) ||
7377 !LIST_EMPTY(&inodedep->id_inowait) ||
7378 !TAILQ_EMPTY(&inodedep->id_inoreflst) ||
7379 !TAILQ_EMPTY(&inodedep->id_inoupdt) ||
7380 !TAILQ_EMPTY(&inodedep->id_newinoupdt) ||
7381 !TAILQ_EMPTY(&inodedep->id_extupdt) ||
7382 !TAILQ_EMPTY(&inodedep->id_newextupdt) ||
7383 !TAILQ_EMPTY(&inodedep->id_freeblklst) ||
7384 inodedep->id_mkdiradd != NULL ||
7385 inodedep->id_nlinkdelta != 0)
7388 * Another process might be in initiate_write_inodeblock_ufs[12]
7389 * trying to allocate memory without holding "Softdep Lock".
7391 if ((inodedep->id_state & IOSTARTED) != 0 &&
7392 inodedep->id_savedino1 == NULL)
7395 if (inodedep->id_state & ONDEPLIST)
7396 LIST_REMOVE(inodedep, id_deps);
7397 inodedep->id_state &= ~ONDEPLIST;
7398 inodedep->id_state |= ALLCOMPLETE;
7399 inodedep->id_bmsafemap = NULL;
7400 if (inodedep->id_state & ONWORKLIST)
7401 WORKLIST_REMOVE(&inodedep->id_list);
7402 if (inodedep->id_savedino1 != NULL) {
7403 free(inodedep->id_savedino1, M_SAVEDINO);
7404 inodedep->id_savedino1 = NULL;
7406 if (free_inodedep(inodedep) == 0)
7407 panic("check_inode_unwritten: busy inode");
7412 * Try to free an inodedep structure. Return 1 if it could be freed.
7415 free_inodedep(inodedep)
7416 struct inodedep *inodedep;
7419 mtx_assert(&lk, MA_OWNED);
7420 if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 ||
7421 (inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE ||
7422 !LIST_EMPTY(&inodedep->id_dirremhd) ||
7423 !LIST_EMPTY(&inodedep->id_pendinghd) ||
7424 !LIST_EMPTY(&inodedep->id_bufwait) ||
7425 !LIST_EMPTY(&inodedep->id_inowait) ||
7426 !TAILQ_EMPTY(&inodedep->id_inoreflst) ||
7427 !TAILQ_EMPTY(&inodedep->id_inoupdt) ||
7428 !TAILQ_EMPTY(&inodedep->id_newinoupdt) ||
7429 !TAILQ_EMPTY(&inodedep->id_extupdt) ||
7430 !TAILQ_EMPTY(&inodedep->id_newextupdt) ||
7431 !TAILQ_EMPTY(&inodedep->id_freeblklst) ||
7432 inodedep->id_mkdiradd != NULL ||
7433 inodedep->id_nlinkdelta != 0 ||
7434 inodedep->id_savedino1 != NULL)
7436 if (inodedep->id_state & ONDEPLIST)
7437 LIST_REMOVE(inodedep, id_deps);
7438 LIST_REMOVE(inodedep, id_hash);
7439 WORKITEM_FREE(inodedep, D_INODEDEP);
7444 * Free the block referenced by a freework structure. The parent freeblks
7445 * structure is released and completed when the final cg bitmap reaches
7446 * the disk. This routine may be freeing a jnewblk which never made it to
7447 * disk in which case we do not have to wait as the operation is undone
7448 * in memory immediately.
7451 freework_freeblock(freework)
7452 struct freework *freework;
7454 struct freeblks *freeblks;
7455 struct jnewblk *jnewblk;
7456 struct ufsmount *ump;
7457 struct workhead wkhd;
7462 mtx_assert(&lk, MA_OWNED);
7464 * Handle partial truncate separately.
7466 if (freework->fw_indir) {
7467 complete_trunc_indir(freework);
7470 freeblks = freework->fw_freeblks;
7471 ump = VFSTOUFS(freeblks->fb_list.wk_mp);
7473 needj = MOUNTEDSUJ(freeblks->fb_list.wk_mp) != 0;
7474 bsize = lfragtosize(fs, freework->fw_frags);
7477 * DEPCOMPLETE is cleared in indirblk_insert() if the block lives
7478 * on the indirblk hashtable and prevents premature freeing.
7480 freework->fw_state |= DEPCOMPLETE;
7482 * SUJ needs to wait for the segment referencing freed indirect
7483 * blocks to expire so that we know the checker will not confuse
7484 * a re-allocated indirect block with its old contents.
7486 if (needj && freework->fw_lbn <= -NDADDR)
7487 indirblk_insert(freework);
7489 * If we are canceling an existing jnewblk pass it to the free
7490 * routine, otherwise pass the freeblk which will ultimately
7491 * release the freeblks. If we're not journaling, we can just
7492 * free the freeblks immediately.
7494 jnewblk = freework->fw_jnewblk;
7495 if (jnewblk != NULL) {
7496 cancel_jnewblk(jnewblk, &wkhd);
7499 freework->fw_state |= DELAYEDFREE;
7500 freeblks->fb_cgwait++;
7501 WORKLIST_INSERT(&wkhd, &freework->fw_list);
7504 freeblks_free(ump, freeblks, btodb(bsize));
7506 "freework_freeblock: ino %d blkno %jd lbn %jd size %ld",
7507 freeblks->fb_inum, freework->fw_blkno, freework->fw_lbn, bsize);
7508 ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, bsize,
7509 freeblks->fb_inum, freeblks->fb_vtype, &wkhd);
7512 * The jnewblk will be discarded and the bits in the map never
7513 * made it to disk. We can immediately free the freeblk.
7516 handle_written_freework(freework);
7520 * We enqueue freework items that need processing back on the freeblks and
7521 * add the freeblks to the worklist. This makes it easier to find all work
7522 * required to flush a truncation in process_truncates().
7525 freework_enqueue(freework)
7526 struct freework *freework;
7528 struct freeblks *freeblks;
7530 freeblks = freework->fw_freeblks;
7531 if ((freework->fw_state & INPROGRESS) == 0)
7532 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list);
7533 if ((freeblks->fb_state &
7534 (ONWORKLIST | INPROGRESS | ALLCOMPLETE)) == ALLCOMPLETE &&
7535 LIST_EMPTY(&freeblks->fb_jblkdephd))
7536 add_to_worklist(&freeblks->fb_list, WK_NODELAY);
7540 * Start, continue, or finish the process of freeing an indirect block tree.
7541 * The free operation may be paused at any point with fw_off containing the
7542 * offset to restart from. This enables us to implement some flow control
7543 * for large truncates which may fan out and generate a huge number of
7547 handle_workitem_indirblk(freework)
7548 struct freework *freework;
7550 struct freeblks *freeblks;
7551 struct ufsmount *ump;
7554 freeblks = freework->fw_freeblks;
7555 ump = VFSTOUFS(freeblks->fb_list.wk_mp);
7557 if (freework->fw_state & DEPCOMPLETE) {
7558 handle_written_freework(freework);
7561 if (freework->fw_off == NINDIR(fs)) {
7562 freework_freeblock(freework);
7565 freework->fw_state |= INPROGRESS;
7567 indir_trunc(freework, fsbtodb(fs, freework->fw_blkno),
7573 * Called when a freework structure attached to a cg buf is written. The
7574 * ref on either the parent or the freeblks structure is released and
7575 * the freeblks is added back to the worklist if there is more work to do.
7578 handle_written_freework(freework)
7579 struct freework *freework;
7581 struct freeblks *freeblks;
7582 struct freework *parent;
7584 freeblks = freework->fw_freeblks;
7585 parent = freework->fw_parent;
7586 if (freework->fw_state & DELAYEDFREE)
7587 freeblks->fb_cgwait--;
7588 freework->fw_state |= COMPLETE;
7589 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE)
7590 WORKITEM_FREE(freework, D_FREEWORK);
7592 if (--parent->fw_ref == 0)
7593 freework_enqueue(parent);
7596 if (--freeblks->fb_ref != 0)
7598 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST | INPROGRESS)) ==
7599 ALLCOMPLETE && LIST_EMPTY(&freeblks->fb_jblkdephd))
7600 add_to_worklist(&freeblks->fb_list, WK_NODELAY);
7604 * This workitem routine performs the block de-allocation.
7605 * The workitem is added to the pending list after the updated
7606 * inode block has been written to disk. As mentioned above,
7607 * checks regarding the number of blocks de-allocated (compared
7608 * to the number of blocks allocated for the file) are also
7609 * performed in this function.
7612 handle_workitem_freeblocks(freeblks, flags)
7613 struct freeblks *freeblks;
7616 struct freework *freework;
7617 struct newblk *newblk;
7618 struct allocindir *aip;
7619 struct ufsmount *ump;
7620 struct worklist *wk;
7622 KASSERT(LIST_EMPTY(&freeblks->fb_jblkdephd),
7623 ("handle_workitem_freeblocks: Journal entries not written."));
7624 ump = VFSTOUFS(freeblks->fb_list.wk_mp);
7626 while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) {
7627 WORKLIST_REMOVE(wk);
7628 switch (wk->wk_type) {
7630 wk->wk_state |= COMPLETE;
7631 add_to_worklist(wk, 0);
7635 free_newblk(WK_NEWBLK(wk));
7639 aip = WK_ALLOCINDIR(wk);
7641 if (aip->ai_state & DELAYEDFREE) {
7643 freework = newfreework(ump, freeblks, NULL,
7644 aip->ai_lbn, aip->ai_newblkno,
7645 ump->um_fs->fs_frag, 0, 0);
7648 newblk = WK_NEWBLK(wk);
7649 if (newblk->nb_jnewblk) {
7650 freework->fw_jnewblk = newblk->nb_jnewblk;
7651 newblk->nb_jnewblk->jn_dep = &freework->fw_list;
7652 newblk->nb_jnewblk = NULL;
7654 free_newblk(newblk);
7658 freework = WK_FREEWORK(wk);
7659 if (freework->fw_lbn <= -NDADDR)
7660 handle_workitem_indirblk(freework);
7662 freework_freeblock(freework);
7665 panic("handle_workitem_freeblocks: Unknown type %s",
7666 TYPENAME(wk->wk_type));
7669 if (freeblks->fb_ref != 0) {
7670 freeblks->fb_state &= ~INPROGRESS;
7671 wake_worklist(&freeblks->fb_list);
7676 return handle_complete_freeblocks(freeblks, flags);
7681 * Handle completion of block free via truncate. This allows fs_pending
7682 * to track the actual free block count more closely than if we only updated
7683 * it at the end. We must be careful to handle cases where the block count
7684 * on free was incorrect.
7687 freeblks_free(ump, freeblks, blocks)
7688 struct ufsmount *ump;
7689 struct freeblks *freeblks;
7693 ufs2_daddr_t remain;
7696 remain = -freeblks->fb_chkcnt;
7697 freeblks->fb_chkcnt += blocks;
7699 if (remain < blocks)
7702 fs->fs_pendingblocks -= blocks;
7708 * Once all of the freework workitems are complete we can retire the
7709 * freeblocks dependency and any journal work awaiting completion. This
7710 * can not be called until all other dependencies are stable on disk.
7713 handle_complete_freeblocks(freeblks, flags)
7714 struct freeblks *freeblks;
7717 struct inodedep *inodedep;
7721 struct ufsmount *ump;
7724 ump = VFSTOUFS(freeblks->fb_list.wk_mp);
7726 flags = LK_EXCLUSIVE | flags;
7727 spare = freeblks->fb_chkcnt;
7730 * If we did not release the expected number of blocks we may have
7731 * to adjust the inode block count here. Only do so if it wasn't
7732 * a truncation to zero and the modrev still matches.
7734 if (spare && freeblks->fb_len != 0) {
7735 if (ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_inum,
7736 flags, &vp, FFSV_FORCEINSMQ) != 0)
7739 if (DIP(ip, i_modrev) == freeblks->fb_modrev) {
7740 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - spare);
7741 ip->i_flag |= IN_CHANGE;
7743 * We must wait so this happens before the
7744 * journal is reclaimed.
7752 fs->fs_pendingblocks += spare;
7758 quotaadj(freeblks->fb_quota, ump, -spare);
7759 quotarele(freeblks->fb_quota);
7762 if (freeblks->fb_state & ONDEPLIST) {
7763 inodedep_lookup(freeblks->fb_list.wk_mp, freeblks->fb_inum,
7765 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, fb_next);
7766 freeblks->fb_state &= ~ONDEPLIST;
7767 if (TAILQ_EMPTY(&inodedep->id_freeblklst))
7768 free_inodedep(inodedep);
7771 * All of the freeblock deps must be complete prior to this call
7772 * so it's now safe to complete earlier outstanding journal entries.
7774 handle_jwork(&freeblks->fb_jwork);
7775 WORKITEM_FREE(freeblks, D_FREEBLKS);
7781 * Release blocks associated with the freeblks and stored in the indirect
7782 * block dbn. If level is greater than SINGLE, the block is an indirect block
7783 * and recursive calls to indirtrunc must be used to cleanse other indirect
7786 * This handles partial and complete truncation of blocks. Partial is noted
7787 * with goingaway == 0. In this case the freework is completed after the
7788 * zero'd indirects are written to disk. For full truncation the freework
7789 * is completed after the block is freed.
7792 indir_trunc(freework, dbn, lbn)
7793 struct freework *freework;
7797 struct freework *nfreework;
7798 struct workhead wkhd;
7799 struct freeblks *freeblks;
7802 struct indirdep *indirdep;
7803 struct ufsmount *ump;
7804 ufs1_daddr_t *bap1 = 0;
7805 ufs2_daddr_t nb, nnb, *bap2 = 0;
7806 ufs_lbn_t lbnadd, nlbn;
7807 int i, nblocks, ufs1fmt;
7815 freeblks = freework->fw_freeblks;
7816 ump = VFSTOUFS(freeblks->fb_list.wk_mp);
7819 * Get buffer of block pointers to be freed. There are three cases:
7821 * 1) Partial truncate caches the indirdep pointer in the freework
7822 * which provides us a back copy to the save bp which holds the
7823 * pointers we want to clear. When this completes the zero
7824 * pointers are written to the real copy.
7825 * 2) The indirect is being completely truncated, cancel_indirdep()
7826 * eliminated the real copy and placed the indirdep on the saved
7827 * copy. The indirdep and buf are discarded when this completes.
7828 * 3) The indirect was not in memory, we read a copy off of the disk
7829 * using the devvp and drop and invalidate the buffer when we're
7834 if (freework->fw_indir != NULL) {
7836 indirdep = freework->fw_indir;
7837 bp = indirdep->ir_savebp;
7838 if (bp == NULL || bp->b_blkno != dbn)
7839 panic("indir_trunc: Bad saved buf %p blkno %jd",
7841 } else if ((bp = incore(&freeblks->fb_devvp->v_bufobj, dbn)) != NULL) {
7843 * The lock prevents the buf dep list from changing and
7844 * indirects on devvp should only ever have one dependency.
7846 indirdep = WK_INDIRDEP(LIST_FIRST(&bp->b_dep));
7847 if (indirdep == NULL || (indirdep->ir_state & GOINGAWAY) == 0)
7848 panic("indir_trunc: Bad indirdep %p from buf %p",
7850 } else if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize,
7851 NOCRED, &bp) != 0) {
7856 /* Protects against a race with complete_trunc_indir(). */
7857 freework->fw_state &= ~INPROGRESS;
7859 * If we have an indirdep we need to enforce the truncation order
7860 * and discard it when it is complete.
7863 if (freework != TAILQ_FIRST(&indirdep->ir_trunc) &&
7864 !TAILQ_EMPTY(&indirdep->ir_trunc)) {
7866 * Add the complete truncate to the list on the
7867 * indirdep to enforce in-order processing.
7869 if (freework->fw_indir == NULL)
7870 TAILQ_INSERT_TAIL(&indirdep->ir_trunc,
7876 * If we're goingaway, free the indirdep. Otherwise it will
7877 * linger until the write completes.
7880 free_indirdep(indirdep);
7881 ump->um_numindirdeps -= 1;
7885 /* Initialize pointers depending on block size. */
7886 if (ump->um_fstype == UFS1) {
7887 bap1 = (ufs1_daddr_t *)bp->b_data;
7888 nb = bap1[freework->fw_off];
7891 bap2 = (ufs2_daddr_t *)bp->b_data;
7892 nb = bap2[freework->fw_off];
7895 level = lbn_level(lbn);
7896 needj = MOUNTEDSUJ(UFSTOVFS(ump)) != 0;
7897 lbnadd = lbn_offset(fs, level);
7898 nblocks = btodb(fs->fs_bsize);
7899 nfreework = freework;
7903 * Reclaim blocks. Traverses into nested indirect levels and
7904 * arranges for the current level to be freed when subordinates
7905 * are free when journaling.
7907 for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) {
7908 if (i != NINDIR(fs) - 1) {
7919 nlbn = (lbn + 1) - (i * lbnadd);
7921 nfreework = newfreework(ump, freeblks, freework,
7922 nlbn, nb, fs->fs_frag, 0, 0);
7925 indir_trunc(nfreework, fsbtodb(fs, nb), nlbn);
7927 struct freedep *freedep;
7930 * Attempt to aggregate freedep dependencies for
7931 * all blocks being released to the same CG.
7935 (nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) {
7936 freedep = newfreedep(freework);
7937 WORKLIST_INSERT_UNLOCKED(&wkhd,
7942 "indir_trunc: ino %d blkno %jd size %ld",
7943 freeblks->fb_inum, nb, fs->fs_bsize);
7944 ffs_blkfree(ump, fs, freeblks->fb_devvp, nb,
7945 fs->fs_bsize, freeblks->fb_inum,
7946 freeblks->fb_vtype, &wkhd);
7950 bp->b_flags |= B_INVAL | B_NOCACHE;
7955 freedblocks = (nblocks * cnt);
7957 freedblocks += nblocks;
7958 freeblks_free(ump, freeblks, freedblocks);
7960 * If we are journaling set up the ref counts and offset so this
7961 * indirect can be completed when its children are free.
7965 freework->fw_off = i;
7966 freework->fw_ref += freedeps;
7967 freework->fw_ref -= NINDIR(fs) + 1;
7969 freeblks->fb_cgwait += freedeps;
7970 if (freework->fw_ref == 0)
7971 freework_freeblock(freework);
7976 * If we're not journaling we can free the indirect now.
7978 dbn = dbtofsb(fs, dbn);
7980 "indir_trunc 2: ino %d blkno %jd size %ld",
7981 freeblks->fb_inum, dbn, fs->fs_bsize);
7982 ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize,
7983 freeblks->fb_inum, freeblks->fb_vtype, NULL);
7984 /* Non SUJ softdep does single-threaded truncations. */
7985 if (freework->fw_blkno == dbn) {
7986 freework->fw_state |= ALLCOMPLETE;
7988 handle_written_freework(freework);
7995 * Cancel an allocindir when it is removed via truncation. When bp is not
7996 * NULL the indirect never appeared on disk and is scheduled to be freed
7997 * independently of the indir so we can more easily track journal work.
8000 cancel_allocindir(aip, bp, freeblks, trunc)
8001 struct allocindir *aip;
8003 struct freeblks *freeblks;
8006 struct indirdep *indirdep;
8007 struct freefrag *freefrag;
8008 struct newblk *newblk;
8010 newblk = (struct newblk *)aip;
8011 LIST_REMOVE(aip, ai_next);
8013 * We must eliminate the pointer in bp if it must be freed on its
8014 * own due to partial truncate or pending journal work.
8016 if (bp && (trunc || newblk->nb_jnewblk)) {
8018 * Clear the pointer and mark the aip to be freed
8019 * directly if it never existed on disk.
8021 aip->ai_state |= DELAYEDFREE;
8022 indirdep = aip->ai_indirdep;
8023 if (indirdep->ir_state & UFS1FMT)
8024 ((ufs1_daddr_t *)bp->b_data)[aip->ai_offset] = 0;
8026 ((ufs2_daddr_t *)bp->b_data)[aip->ai_offset] = 0;
8029 * When truncating the previous pointer will be freed via
8030 * savedbp. Eliminate the freefrag which would dup free.
8032 if (trunc && (freefrag = newblk->nb_freefrag) != NULL) {
8033 newblk->nb_freefrag = NULL;
8034 if (freefrag->ff_jdep)
8036 WK_JFREEFRAG(freefrag->ff_jdep));
8037 jwork_move(&freeblks->fb_jwork, &freefrag->ff_jwork);
8038 WORKITEM_FREE(freefrag, D_FREEFRAG);
8041 * If the journal hasn't been written the jnewblk must be passed
8042 * to the call to ffs_blkfree that reclaims the space. We accomplish
8043 * this by leaving the journal dependency on the newblk to be freed
8044 * when a freework is created in handle_workitem_freeblocks().
8046 cancel_newblk(newblk, NULL, &freeblks->fb_jwork);
8047 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list);
8051 * Create the mkdir dependencies for . and .. in a new directory. Link them
8052 * in to a newdirblk so any subsequent additions are tracked properly. The
8053 * caller is responsible for adding the mkdir1 dependency to the journal
8054 * and updating id_mkdiradd. This function returns with lk held.
8056 static struct mkdir *
8057 setup_newdir(dap, newinum, dinum, newdirbp, mkdirp)
8061 struct buf *newdirbp;
8062 struct mkdir **mkdirp;
8064 struct newblk *newblk;
8065 struct pagedep *pagedep;
8066 struct inodedep *inodedep;
8067 struct newdirblk *newdirblk = 0;
8068 struct mkdir *mkdir1, *mkdir2;
8069 struct worklist *wk;
8070 struct jaddref *jaddref;
8073 mp = dap->da_list.wk_mp;
8074 newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK,
8076 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp);
8077 LIST_INIT(&newdirblk->db_mkdir);
8078 mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS);
8079 workitem_alloc(&mkdir1->md_list, D_MKDIR, mp);
8080 mkdir1->md_state = ATTACHED | MKDIR_BODY;
8081 mkdir1->md_diradd = dap;
8082 mkdir1->md_jaddref = NULL;
8083 mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS);
8084 workitem_alloc(&mkdir2->md_list, D_MKDIR, mp);
8085 mkdir2->md_state = ATTACHED | MKDIR_PARENT;
8086 mkdir2->md_diradd = dap;
8087 mkdir2->md_jaddref = NULL;
8088 if (MOUNTEDSUJ(mp) == 0) {
8089 mkdir1->md_state |= DEPCOMPLETE;
8090 mkdir2->md_state |= DEPCOMPLETE;
8093 * Dependency on "." and ".." being written to disk.
8095 mkdir1->md_buf = newdirbp;
8097 LIST_INSERT_HEAD(&mkdirlisthd, mkdir1, md_mkdirs);
8099 * We must link the pagedep, allocdirect, and newdirblk for
8100 * the initial file page so the pointer to the new directory
8101 * is not written until the directory contents are live and
8102 * any subsequent additions are not marked live until the
8103 * block is reachable via the inode.
8105 if (pagedep_lookup(mp, newdirbp, newinum, 0, 0, &pagedep) == 0)
8106 panic("setup_newdir: lost pagedep");
8107 LIST_FOREACH(wk, &newdirbp->b_dep, wk_list)
8108 if (wk->wk_type == D_ALLOCDIRECT)
8111 panic("setup_newdir: lost allocdirect");
8112 if (pagedep->pd_state & NEWBLOCK)
8113 panic("setup_newdir: NEWBLOCK already set");
8114 newblk = WK_NEWBLK(wk);
8115 pagedep->pd_state |= NEWBLOCK;
8116 pagedep->pd_newdirblk = newdirblk;
8117 newdirblk->db_pagedep = pagedep;
8118 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list);
8119 WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list);
8121 * Look up the inodedep for the parent directory so that we
8122 * can link mkdir2 into the pending dotdot jaddref or
8123 * the inode write if there is none. If the inode is
8124 * ALLCOMPLETE and no jaddref is present all dependencies have
8125 * been satisfied and mkdir2 can be freed.
8127 inodedep_lookup(mp, dinum, 0, &inodedep);
8128 if (MOUNTEDSUJ(mp)) {
8129 if (inodedep == NULL)
8130 panic("setup_newdir: Lost parent.");
8131 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
8133 KASSERT(jaddref != NULL && jaddref->ja_parent == newinum &&
8134 (jaddref->ja_state & MKDIR_PARENT),
8135 ("setup_newdir: bad dotdot jaddref %p", jaddref));
8136 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs);
8137 mkdir2->md_jaddref = jaddref;
8138 jaddref->ja_mkdir = mkdir2;
8139 } else if (inodedep == NULL ||
8140 (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
8141 dap->da_state &= ~MKDIR_PARENT;
8142 WORKITEM_FREE(mkdir2, D_MKDIR);
8144 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs);
8145 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir2->md_list);
8153 * Directory entry addition dependencies.
8155 * When adding a new directory entry, the inode (with its incremented link
8156 * count) must be written to disk before the directory entry's pointer to it.
8157 * Also, if the inode is newly allocated, the corresponding freemap must be
8158 * updated (on disk) before the directory entry's pointer. These requirements
8159 * are met via undo/redo on the directory entry's pointer, which consists
8160 * simply of the inode number.
8162 * As directory entries are added and deleted, the free space within a
8163 * directory block can become fragmented. The ufs filesystem will compact
8164 * a fragmented directory block to make space for a new entry. When this
8165 * occurs, the offsets of previously added entries change. Any "diradd"
8166 * dependency structures corresponding to these entries must be updated with
8171 * This routine is called after the in-memory inode's link
8172 * count has been incremented, but before the directory entry's
8173 * pointer to the inode has been set.
8176 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
8177 struct buf *bp; /* buffer containing directory block */
8178 struct inode *dp; /* inode for directory */
8179 off_t diroffset; /* offset of new entry in directory */
8180 ino_t newinum; /* inode referenced by new directory entry */
8181 struct buf *newdirbp; /* non-NULL => contents of new mkdir */
8182 int isnewblk; /* entry is in a newly allocated block */
8184 int offset; /* offset of new entry within directory block */
8185 ufs_lbn_t lbn; /* block in directory containing new entry */
8188 struct newblk *newblk;
8189 struct pagedep *pagedep;
8190 struct inodedep *inodedep;
8191 struct newdirblk *newdirblk = 0;
8192 struct mkdir *mkdir1, *mkdir2;
8193 struct jaddref *jaddref;
8198 * Whiteouts have no dependencies.
8200 if (newinum == WINO) {
8201 if (newdirbp != NULL)
8206 mkdir1 = mkdir2 = NULL;
8207 mp = UFSTOVFS(dp->i_ump);
8209 lbn = lblkno(fs, diroffset);
8210 offset = blkoff(fs, diroffset);
8211 dap = malloc(sizeof(struct diradd), M_DIRADD,
8212 M_SOFTDEP_FLAGS|M_ZERO);
8213 workitem_alloc(&dap->da_list, D_DIRADD, mp);
8214 dap->da_offset = offset;
8215 dap->da_newinum = newinum;
8216 dap->da_state = ATTACHED;
8217 LIST_INIT(&dap->da_jwork);
8218 isindir = bp->b_lblkno >= NDADDR;
8220 (isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) {
8221 newdirblk = malloc(sizeof(struct newdirblk),
8222 M_NEWDIRBLK, M_SOFTDEP_FLAGS);
8223 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp);
8224 LIST_INIT(&newdirblk->db_mkdir);
8227 * If we're creating a new directory setup the dependencies and set
8228 * the dap state to wait for them. Otherwise it's COMPLETE and
8231 if (newdirbp == NULL) {
8232 dap->da_state |= DEPCOMPLETE;
8235 dap->da_state |= MKDIR_BODY | MKDIR_PARENT;
8236 mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp,
8240 * Link into parent directory pagedep to await its being written.
8242 pagedep_lookup(mp, bp, dp->i_number, lbn, DEPALLOC, &pagedep);
8244 if (diradd_lookup(pagedep, offset) != NULL)
8245 panic("softdep_setup_directory_add: %p already at off %d\n",
8246 diradd_lookup(pagedep, offset), offset);
8248 dap->da_pagedep = pagedep;
8249 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap,
8251 inodedep_lookup(mp, newinum, DEPALLOC | NODELAY, &inodedep);
8253 * If we're journaling, link the diradd into the jaddref so it
8254 * may be completed after the journal entry is written. Otherwise,
8255 * link the diradd into its inodedep. If the inode is not yet
8256 * written place it on the bufwait list, otherwise do the post-inode
8257 * write processing to put it on the id_pendinghd list.
8259 if (MOUNTEDSUJ(mp)) {
8260 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
8262 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
8263 ("softdep_setup_directory_add: bad jaddref %p", jaddref));
8264 jaddref->ja_diroff = diroffset;
8265 jaddref->ja_diradd = dap;
8266 add_to_journal(&jaddref->ja_list);
8267 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE)
8268 diradd_inode_written(dap, inodedep);
8270 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
8272 * Add the journal entries for . and .. links now that the primary
8275 if (mkdir1 != NULL && MOUNTEDSUJ(mp)) {
8276 jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref,
8277 inoreflst, if_deps);
8278 KASSERT(jaddref != NULL &&
8279 jaddref->ja_ino == jaddref->ja_parent &&
8280 (jaddref->ja_state & MKDIR_BODY),
8281 ("softdep_setup_directory_add: bad dot jaddref %p",
8283 mkdir1->md_jaddref = jaddref;
8284 jaddref->ja_mkdir = mkdir1;
8286 * It is important that the dotdot journal entry
8287 * is added prior to the dot entry since dot writes
8288 * both the dot and dotdot links. These both must
8289 * be added after the primary link for the journal
8290 * to remain consistent.
8292 add_to_journal(&mkdir2->md_jaddref->ja_list);
8293 add_to_journal(&jaddref->ja_list);
8296 * If we are adding a new directory remember this diradd so that if
8297 * we rename it we can keep the dot and dotdot dependencies. If
8298 * we are adding a new name for an inode that has a mkdiradd we
8299 * must be in rename and we have to move the dot and dotdot
8300 * dependencies to this new name. The old name is being orphaned
8303 if (mkdir1 != NULL) {
8304 if (inodedep->id_mkdiradd != NULL)
8305 panic("softdep_setup_directory_add: Existing mkdir");
8306 inodedep->id_mkdiradd = dap;
8307 } else if (inodedep->id_mkdiradd)
8308 merge_diradd(inodedep, dap);
8311 * There is nothing to do if we are already tracking
8314 if ((pagedep->pd_state & NEWBLOCK) != 0) {
8315 WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
8319 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk)
8321 panic("softdep_setup_directory_add: lost entry");
8322 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list);
8323 pagedep->pd_state |= NEWBLOCK;
8324 pagedep->pd_newdirblk = newdirblk;
8325 newdirblk->db_pagedep = pagedep;
8328 * If we extended into an indirect signal direnter to sync.
8339 * This procedure is called to change the offset of a directory
8340 * entry when compacting a directory block which must be owned
8341 * exclusively by the caller. Note that the actual entry movement
8342 * must be done in this procedure to ensure that no I/O completions
8343 * occur while the move is in progress.
8346 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize)
8347 struct buf *bp; /* Buffer holding directory block. */
8348 struct inode *dp; /* inode for directory */
8349 caddr_t base; /* address of dp->i_offset */
8350 caddr_t oldloc; /* address of old directory location */
8351 caddr_t newloc; /* address of new directory location */
8352 int entrysize; /* size of directory entry */
8354 int offset, oldoffset, newoffset;
8355 struct pagedep *pagedep;
8356 struct jmvref *jmvref;
8363 mp = UFSTOVFS(dp->i_ump);
8364 de = (struct direct *)oldloc;
8368 * Moves are always journaled as it would be too complex to
8369 * determine if any affected adds or removes are present in the
8372 if (MOUNTEDSUJ(mp)) {
8374 jmvref = newjmvref(dp, de->d_ino,
8375 dp->i_offset + (oldloc - base),
8376 dp->i_offset + (newloc - base));
8378 lbn = lblkno(dp->i_fs, dp->i_offset);
8379 offset = blkoff(dp->i_fs, dp->i_offset);
8380 oldoffset = offset + (oldloc - base);
8381 newoffset = offset + (newloc - base);
8383 if (pagedep_lookup(mp, bp, dp->i_number, lbn, flags, &pagedep) == 0)
8385 dap = diradd_lookup(pagedep, oldoffset);
8387 dap->da_offset = newoffset;
8388 newoffset = DIRADDHASH(newoffset);
8389 oldoffset = DIRADDHASH(oldoffset);
8390 if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE &&
8391 newoffset != oldoffset) {
8392 LIST_REMOVE(dap, da_pdlist);
8393 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset],
8399 jmvref->jm_pagedep = pagedep;
8400 LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps);
8401 add_to_journal(&jmvref->jm_list);
8403 bcopy(oldloc, newloc, entrysize);
8408 * Move the mkdir dependencies and journal work from one diradd to another
8409 * when renaming a directory. The new name must depend on the mkdir deps
8410 * completing as the old name did. Directories can only have one valid link
8411 * at a time so one must be canonical.
8414 merge_diradd(inodedep, newdap)
8415 struct inodedep *inodedep;
8416 struct diradd *newdap;
8418 struct diradd *olddap;
8419 struct mkdir *mkdir, *nextmd;
8422 olddap = inodedep->id_mkdiradd;
8423 inodedep->id_mkdiradd = newdap;
8424 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
8425 newdap->da_state &= ~DEPCOMPLETE;
8426 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) {
8427 nextmd = LIST_NEXT(mkdir, md_mkdirs);
8428 if (mkdir->md_diradd != olddap)
8430 mkdir->md_diradd = newdap;
8431 state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY);
8432 newdap->da_state |= state;
8433 olddap->da_state &= ~state;
8434 if ((olddap->da_state &
8435 (MKDIR_PARENT | MKDIR_BODY)) == 0)
8438 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0)
8439 panic("merge_diradd: unfound ref");
8442 * Any mkdir related journal items are not safe to be freed until
8443 * the new name is stable.
8445 jwork_move(&newdap->da_jwork, &olddap->da_jwork);
8446 olddap->da_state |= DEPCOMPLETE;
8447 complete_diradd(olddap);
8451 * Move the diradd to the pending list when all diradd dependencies are
8455 complete_diradd(dap)
8458 struct pagedep *pagedep;
8460 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
8461 if (dap->da_state & DIRCHG)
8462 pagedep = dap->da_previous->dm_pagedep;
8464 pagedep = dap->da_pagedep;
8465 LIST_REMOVE(dap, da_pdlist);
8466 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
8471 * Cancel a diradd when a dirrem overlaps with it. We must cancel the journal
8472 * add entries and conditonally journal the remove.
8475 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref)
8477 struct dirrem *dirrem;
8478 struct jremref *jremref;
8479 struct jremref *dotremref;
8480 struct jremref *dotdotremref;
8482 struct inodedep *inodedep;
8483 struct jaddref *jaddref;
8484 struct inoref *inoref;
8485 struct mkdir *mkdir;
8488 * If no remove references were allocated we're on a non-journaled
8489 * filesystem and can skip the cancel step.
8491 if (jremref == NULL) {
8492 free_diradd(dap, NULL);
8496 * Cancel the primary name an free it if it does not require
8499 if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum,
8500 0, &inodedep) != 0) {
8501 /* Abort the addref that reference this diradd. */
8502 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
8503 if (inoref->if_list.wk_type != D_JADDREF)
8505 jaddref = (struct jaddref *)inoref;
8506 if (jaddref->ja_diradd != dap)
8508 if (cancel_jaddref(jaddref, inodedep,
8509 &dirrem->dm_jwork) == 0) {
8510 free_jremref(jremref);
8517 * Cancel subordinate names and free them if they do not require
8520 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
8521 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) {
8522 if (mkdir->md_diradd != dap)
8524 if ((jaddref = mkdir->md_jaddref) == NULL)
8526 mkdir->md_jaddref = NULL;
8527 if (mkdir->md_state & MKDIR_PARENT) {
8528 if (cancel_jaddref(jaddref, NULL,
8529 &dirrem->dm_jwork) == 0) {
8530 free_jremref(dotdotremref);
8531 dotdotremref = NULL;
8534 if (cancel_jaddref(jaddref, inodedep,
8535 &dirrem->dm_jwork) == 0) {
8536 free_jremref(dotremref);
8544 journal_jremref(dirrem, jremref, inodedep);
8546 journal_jremref(dirrem, dotremref, inodedep);
8548 journal_jremref(dirrem, dotdotremref, NULL);
8549 jwork_move(&dirrem->dm_jwork, &dap->da_jwork);
8550 free_diradd(dap, &dirrem->dm_jwork);
8554 * Free a diradd dependency structure. This routine must be called
8555 * with splbio interrupts blocked.
8558 free_diradd(dap, wkhd)
8560 struct workhead *wkhd;
8562 struct dirrem *dirrem;
8563 struct pagedep *pagedep;
8564 struct inodedep *inodedep;
8565 struct mkdir *mkdir, *nextmd;
8567 mtx_assert(&lk, MA_OWNED);
8568 LIST_REMOVE(dap, da_pdlist);
8569 if (dap->da_state & ONWORKLIST)
8570 WORKLIST_REMOVE(&dap->da_list);
8571 if ((dap->da_state & DIRCHG) == 0) {
8572 pagedep = dap->da_pagedep;
8574 dirrem = dap->da_previous;
8575 pagedep = dirrem->dm_pagedep;
8576 dirrem->dm_dirinum = pagedep->pd_ino;
8577 dirrem->dm_state |= COMPLETE;
8578 if (LIST_EMPTY(&dirrem->dm_jremrefhd))
8579 add_to_worklist(&dirrem->dm_list, 0);
8581 if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum,
8583 if (inodedep->id_mkdiradd == dap)
8584 inodedep->id_mkdiradd = NULL;
8585 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
8586 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) {
8587 nextmd = LIST_NEXT(mkdir, md_mkdirs);
8588 if (mkdir->md_diradd != dap)
8591 ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY));
8592 LIST_REMOVE(mkdir, md_mkdirs);
8593 if (mkdir->md_state & ONWORKLIST)
8594 WORKLIST_REMOVE(&mkdir->md_list);
8595 if (mkdir->md_jaddref != NULL)
8596 panic("free_diradd: Unexpected jaddref");
8597 WORKITEM_FREE(mkdir, D_MKDIR);
8598 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0)
8601 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0)
8602 panic("free_diradd: unfound ref");
8605 free_inodedep(inodedep);
8607 * Free any journal segments waiting for the directory write.
8609 handle_jwork(&dap->da_jwork);
8610 WORKITEM_FREE(dap, D_DIRADD);
8614 * Directory entry removal dependencies.
8616 * When removing a directory entry, the entry's inode pointer must be
8617 * zero'ed on disk before the corresponding inode's link count is decremented
8618 * (possibly freeing the inode for re-use). This dependency is handled by
8619 * updating the directory entry but delaying the inode count reduction until
8620 * after the directory block has been written to disk. After this point, the
8621 * inode count can be decremented whenever it is convenient.
8625 * This routine should be called immediately after removing
8626 * a directory entry. The inode's link count should not be
8627 * decremented by the calling procedure -- the soft updates
8628 * code will do this task when it is safe.
8631 softdep_setup_remove(bp, dp, ip, isrmdir)
8632 struct buf *bp; /* buffer containing directory block */
8633 struct inode *dp; /* inode for the directory being modified */
8634 struct inode *ip; /* inode for directory entry being removed */
8635 int isrmdir; /* indicates if doing RMDIR */
8637 struct dirrem *dirrem, *prevdirrem;
8638 struct inodedep *inodedep;
8642 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want
8643 * newdirrem() to setup the full directory remove which requires
8646 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
8648 * Add the dirrem to the inodedep's pending remove list for quick
8651 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0,
8653 panic("softdep_setup_remove: Lost inodedep.");
8654 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked"));
8655 dirrem->dm_state |= ONDEPLIST;
8656 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
8659 * If the COMPLETE flag is clear, then there were no active
8660 * entries and we want to roll back to a zeroed entry until
8661 * the new inode is committed to disk. If the COMPLETE flag is
8662 * set then we have deleted an entry that never made it to
8663 * disk. If the entry we deleted resulted from a name change,
8664 * then the old name still resides on disk. We cannot delete
8665 * its inode (returned to us in prevdirrem) until the zeroed
8666 * directory entry gets to disk. The new inode has never been
8667 * referenced on the disk, so can be deleted immediately.
8669 if ((dirrem->dm_state & COMPLETE) == 0) {
8670 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem,
8674 if (prevdirrem != NULL)
8675 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd,
8676 prevdirrem, dm_next);
8677 dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino;
8678 direct = LIST_EMPTY(&dirrem->dm_jremrefhd);
8681 handle_workitem_remove(dirrem, 0);
8686 * Check for an entry matching 'offset' on both the pd_dirraddhd list and the
8687 * pd_pendinghd list of a pagedep.
8689 static struct diradd *
8690 diradd_lookup(pagedep, offset)
8691 struct pagedep *pagedep;
8696 LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist)
8697 if (dap->da_offset == offset)
8699 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist)
8700 if (dap->da_offset == offset)
8706 * Search for a .. diradd dependency in a directory that is being removed.
8707 * If the directory was renamed to a new parent we have a diradd rather
8708 * than a mkdir for the .. entry. We need to cancel it now before
8709 * it is found in truncate().
8711 static struct jremref *
8712 cancel_diradd_dotdot(ip, dirrem, jremref)
8714 struct dirrem *dirrem;
8715 struct jremref *jremref;
8717 struct pagedep *pagedep;
8719 struct worklist *wk;
8721 if (pagedep_lookup(UFSTOVFS(ip->i_ump), NULL, ip->i_number, 0, 0,
8724 dap = diradd_lookup(pagedep, DOTDOT_OFFSET);
8727 cancel_diradd(dap, dirrem, jremref, NULL, NULL);
8729 * Mark any journal work as belonging to the parent so it is freed
8730 * with the .. reference.
8732 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list)
8733 wk->wk_state |= MKDIR_PARENT;
8738 * Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to
8739 * replace it with a dirrem/diradd pair as a result of re-parenting a
8740 * directory. This ensures that we don't simultaneously have a mkdir and
8741 * a diradd for the same .. entry.
8743 static struct jremref *
8744 cancel_mkdir_dotdot(ip, dirrem, jremref)
8746 struct dirrem *dirrem;
8747 struct jremref *jremref;
8749 struct inodedep *inodedep;
8750 struct jaddref *jaddref;
8751 struct mkdir *mkdir;
8754 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0,
8757 dap = inodedep->id_mkdiradd;
8758 if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0)
8760 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir;
8761 mkdir = LIST_NEXT(mkdir, md_mkdirs))
8762 if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT)
8765 panic("cancel_mkdir_dotdot: Unable to find mkdir\n");
8766 if ((jaddref = mkdir->md_jaddref) != NULL) {
8767 mkdir->md_jaddref = NULL;
8768 jaddref->ja_state &= ~MKDIR_PARENT;
8769 if (inodedep_lookup(UFSTOVFS(ip->i_ump), jaddref->ja_ino, 0,
8771 panic("cancel_mkdir_dotdot: Lost parent inodedep");
8772 if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) {
8773 journal_jremref(dirrem, jremref, inodedep);
8777 if (mkdir->md_state & ONWORKLIST)
8778 WORKLIST_REMOVE(&mkdir->md_list);
8779 mkdir->md_state |= ALLCOMPLETE;
8780 complete_mkdir(mkdir);
8785 journal_jremref(dirrem, jremref, inodedep)
8786 struct dirrem *dirrem;
8787 struct jremref *jremref;
8788 struct inodedep *inodedep;
8791 if (inodedep == NULL)
8792 if (inodedep_lookup(jremref->jr_list.wk_mp,
8793 jremref->jr_ref.if_ino, 0, &inodedep) == 0)
8794 panic("journal_jremref: Lost inodedep");
8795 LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps);
8796 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps);
8797 add_to_journal(&jremref->jr_list);
8801 dirrem_journal(dirrem, jremref, dotremref, dotdotremref)
8802 struct dirrem *dirrem;
8803 struct jremref *jremref;
8804 struct jremref *dotremref;
8805 struct jremref *dotdotremref;
8807 struct inodedep *inodedep;
8810 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0,
8812 panic("dirrem_journal: Lost inodedep");
8813 journal_jremref(dirrem, jremref, inodedep);
8815 journal_jremref(dirrem, dotremref, inodedep);
8817 journal_jremref(dirrem, dotdotremref, NULL);
8821 * Allocate a new dirrem if appropriate and return it along with
8822 * its associated pagedep. Called without a lock, returns with lock.
8824 static struct dirrem *
8825 newdirrem(bp, dp, ip, isrmdir, prevdirremp)
8826 struct buf *bp; /* buffer containing directory block */
8827 struct inode *dp; /* inode for the directory being modified */
8828 struct inode *ip; /* inode for directory entry being removed */
8829 int isrmdir; /* indicates if doing RMDIR */
8830 struct dirrem **prevdirremp; /* previously referenced inode, if any */
8835 struct dirrem *dirrem;
8836 struct pagedep *pagedep;
8837 struct jremref *jremref;
8838 struct jremref *dotremref;
8839 struct jremref *dotdotremref;
8843 * Whiteouts have no deletion dependencies.
8846 panic("newdirrem: whiteout");
8849 * If we are over our limit, try to improve the situation.
8850 * Limiting the number of dirrem structures will also limit
8851 * the number of freefile and freeblks structures.
8854 if (!IS_SNAPSHOT(ip) && dep_current[D_DIRREM] > max_softdeps / 2)
8855 (void) request_cleanup(ITOV(dp)->v_mount, FLUSH_BLOCKS);
8857 dirrem = malloc(sizeof(struct dirrem),
8858 M_DIRREM, M_SOFTDEP_FLAGS|M_ZERO);
8859 workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount);
8860 LIST_INIT(&dirrem->dm_jremrefhd);
8861 LIST_INIT(&dirrem->dm_jwork);
8862 dirrem->dm_state = isrmdir ? RMDIR : 0;
8863 dirrem->dm_oldinum = ip->i_number;
8864 *prevdirremp = NULL;
8866 * Allocate remove reference structures to track journal write
8867 * dependencies. We will always have one for the link and
8868 * when doing directories we will always have one more for dot.
8869 * When renaming a directory we skip the dotdot link change so
8870 * this is not needed.
8872 jremref = dotremref = dotdotremref = NULL;
8873 if (DOINGSUJ(dvp)) {
8875 jremref = newjremref(dirrem, dp, ip, dp->i_offset,
8876 ip->i_effnlink + 2);
8877 dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET,
8878 ip->i_effnlink + 1);
8879 dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET,
8880 dp->i_effnlink + 1);
8881 dotdotremref->jr_state |= MKDIR_PARENT;
8883 jremref = newjremref(dirrem, dp, ip, dp->i_offset,
8884 ip->i_effnlink + 1);
8887 lbn = lblkno(dp->i_fs, dp->i_offset);
8888 offset = blkoff(dp->i_fs, dp->i_offset);
8889 pagedep_lookup(UFSTOVFS(dp->i_ump), bp, dp->i_number, lbn, DEPALLOC,
8891 dirrem->dm_pagedep = pagedep;
8892 dirrem->dm_offset = offset;
8894 * If we're renaming a .. link to a new directory, cancel any
8895 * existing MKDIR_PARENT mkdir. If it has already been canceled
8896 * the jremref is preserved for any potential diradd in this
8897 * location. This can not coincide with a rmdir.
8899 if (dp->i_offset == DOTDOT_OFFSET) {
8901 panic("newdirrem: .. directory change during remove?");
8902 jremref = cancel_mkdir_dotdot(dp, dirrem, jremref);
8905 * If we're removing a directory search for the .. dependency now and
8906 * cancel it. Any pending journal work will be added to the dirrem
8907 * to be completed when the workitem remove completes.
8910 dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref);
8912 * Check for a diradd dependency for the same directory entry.
8913 * If present, then both dependencies become obsolete and can
8916 dap = diradd_lookup(pagedep, offset);
8919 * Link the jremref structures into the dirrem so they are
8920 * written prior to the pagedep.
8923 dirrem_journal(dirrem, jremref, dotremref,
8928 * Must be ATTACHED at this point.
8930 if ((dap->da_state & ATTACHED) == 0)
8931 panic("newdirrem: not ATTACHED");
8932 if (dap->da_newinum != ip->i_number)
8933 panic("newdirrem: inum %d should be %d",
8934 ip->i_number, dap->da_newinum);
8936 * If we are deleting a changed name that never made it to disk,
8937 * then return the dirrem describing the previous inode (which
8938 * represents the inode currently referenced from this entry on disk).
8940 if ((dap->da_state & DIRCHG) != 0) {
8941 *prevdirremp = dap->da_previous;
8942 dap->da_state &= ~DIRCHG;
8943 dap->da_pagedep = pagedep;
8946 * We are deleting an entry that never made it to disk.
8947 * Mark it COMPLETE so we can delete its inode immediately.
8949 dirrem->dm_state |= COMPLETE;
8950 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref);
8953 struct worklist *wk;
8955 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list)
8956 if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT))
8957 panic("bad wk %p (0x%X)\n", wk, wk->wk_state);
8965 * Directory entry change dependencies.
8967 * Changing an existing directory entry requires that an add operation
8968 * be completed first followed by a deletion. The semantics for the addition
8969 * are identical to the description of adding a new entry above except
8970 * that the rollback is to the old inode number rather than zero. Once
8971 * the addition dependency is completed, the removal is done as described
8972 * in the removal routine above.
8976 * This routine should be called immediately after changing
8977 * a directory entry. The inode's link count should not be
8978 * decremented by the calling procedure -- the soft updates
8979 * code will perform this task when it is safe.
8982 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
8983 struct buf *bp; /* buffer containing directory block */
8984 struct inode *dp; /* inode for the directory being modified */
8985 struct inode *ip; /* inode for directory entry being removed */
8986 ino_t newinum; /* new inode number for changed entry */
8987 int isrmdir; /* indicates if doing RMDIR */
8990 struct diradd *dap = NULL;
8991 struct dirrem *dirrem, *prevdirrem;
8992 struct pagedep *pagedep;
8993 struct inodedep *inodedep;
8994 struct jaddref *jaddref;
8997 offset = blkoff(dp->i_fs, dp->i_offset);
8998 mp = UFSTOVFS(dp->i_ump);
9001 * Whiteouts do not need diradd dependencies.
9003 if (newinum != WINO) {
9004 dap = malloc(sizeof(struct diradd),
9005 M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO);
9006 workitem_alloc(&dap->da_list, D_DIRADD, mp);
9007 dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE;
9008 dap->da_offset = offset;
9009 dap->da_newinum = newinum;
9010 LIST_INIT(&dap->da_jwork);
9014 * Allocate a new dirrem and ACQUIRE_LOCK.
9016 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
9017 pagedep = dirrem->dm_pagedep;
9019 * The possible values for isrmdir:
9020 * 0 - non-directory file rename
9021 * 1 - directory rename within same directory
9022 * inum - directory rename to new directory of given inode number
9023 * When renaming to a new directory, we are both deleting and
9024 * creating a new directory entry, so the link count on the new
9025 * directory should not change. Thus we do not need the followup
9026 * dirrem which is usually done in handle_workitem_remove. We set
9027 * the DIRCHG flag to tell handle_workitem_remove to skip the
9031 dirrem->dm_state |= DIRCHG;
9034 * Whiteouts have no additional dependencies,
9035 * so just put the dirrem on the correct list.
9037 if (newinum == WINO) {
9038 if ((dirrem->dm_state & COMPLETE) == 0) {
9039 LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem,
9042 dirrem->dm_dirinum = pagedep->pd_ino;
9043 if (LIST_EMPTY(&dirrem->dm_jremrefhd))
9044 add_to_worklist(&dirrem->dm_list, 0);
9050 * Add the dirrem to the inodedep's pending remove list for quick
9051 * discovery later. A valid nlinkdelta ensures that this lookup
9054 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0)
9055 panic("softdep_setup_directory_change: Lost inodedep.");
9056 dirrem->dm_state |= ONDEPLIST;
9057 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
9060 * If the COMPLETE flag is clear, then there were no active
9061 * entries and we want to roll back to the previous inode until
9062 * the new inode is committed to disk. If the COMPLETE flag is
9063 * set, then we have deleted an entry that never made it to disk.
9064 * If the entry we deleted resulted from a name change, then the old
9065 * inode reference still resides on disk. Any rollback that we do
9066 * needs to be to that old inode (returned to us in prevdirrem). If
9067 * the entry we deleted resulted from a create, then there is
9068 * no entry on the disk, so we want to roll back to zero rather
9069 * than the uncommitted inode. In either of the COMPLETE cases we
9070 * want to immediately free the unwritten and unreferenced inode.
9072 if ((dirrem->dm_state & COMPLETE) == 0) {
9073 dap->da_previous = dirrem;
9075 if (prevdirrem != NULL) {
9076 dap->da_previous = prevdirrem;
9078 dap->da_state &= ~DIRCHG;
9079 dap->da_pagedep = pagedep;
9081 dirrem->dm_dirinum = pagedep->pd_ino;
9082 if (LIST_EMPTY(&dirrem->dm_jremrefhd))
9083 add_to_worklist(&dirrem->dm_list, 0);
9086 * Lookup the jaddref for this journal entry. We must finish
9087 * initializing it and make the diradd write dependent on it.
9088 * If we're not journaling, put it on the id_bufwait list if the
9089 * inode is not yet written. If it is written, do the post-inode
9090 * write processing to put it on the id_pendinghd list.
9092 inodedep_lookup(mp, newinum, DEPALLOC | NODELAY, &inodedep);
9093 if (MOUNTEDSUJ(mp)) {
9094 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
9096 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
9097 ("softdep_setup_directory_change: bad jaddref %p",
9099 jaddref->ja_diroff = dp->i_offset;
9100 jaddref->ja_diradd = dap;
9101 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)],
9103 add_to_journal(&jaddref->ja_list);
9104 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
9105 dap->da_state |= COMPLETE;
9106 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
9107 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
9109 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)],
9111 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
9114 * If we're making a new name for a directory that has not been
9115 * committed when need to move the dot and dotdot references to
9118 if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET)
9119 merge_diradd(inodedep, dap);
9124 * Called whenever the link count on an inode is changed.
9125 * It creates an inode dependency so that the new reference(s)
9126 * to the inode cannot be committed to disk until the updated
9127 * inode has been written.
9130 softdep_change_linkcnt(ip)
9131 struct inode *ip; /* the inode with the increased link count */
9133 struct inodedep *inodedep;
9138 if (IS_SNAPSHOT(ip))
9140 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, dflags, &inodedep);
9141 if (ip->i_nlink < ip->i_effnlink)
9142 panic("softdep_change_linkcnt: bad delta");
9143 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
9148 * Attach a sbdep dependency to the superblock buf so that we can keep
9149 * track of the head of the linked list of referenced but unlinked inodes.
9152 softdep_setup_sbupdate(ump, fs, bp)
9153 struct ufsmount *ump;
9157 struct sbdep *sbdep;
9158 struct worklist *wk;
9160 if (MOUNTEDSUJ(UFSTOVFS(ump)) == 0)
9162 LIST_FOREACH(wk, &bp->b_dep, wk_list)
9163 if (wk->wk_type == D_SBDEP)
9167 sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS);
9168 workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump));
9170 sbdep->sb_ump = ump;
9172 WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list);
9177 * Return the first unlinked inodedep which is ready to be the head of the
9178 * list. The inodedep and all those after it must have valid next pointers.
9180 static struct inodedep *
9181 first_unlinked_inodedep(ump)
9182 struct ufsmount *ump;
9184 struct inodedep *inodedep;
9185 struct inodedep *idp;
9187 mtx_assert(&lk, MA_OWNED);
9188 for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst);
9189 inodedep; inodedep = idp) {
9190 if ((inodedep->id_state & UNLINKNEXT) == 0)
9192 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
9193 if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0)
9195 if ((inodedep->id_state & UNLINKPREV) == 0)
9202 * Set the sujfree unlinked head pointer prior to writing a superblock.
9205 initiate_write_sbdep(sbdep)
9206 struct sbdep *sbdep;
9208 struct inodedep *inodedep;
9212 bpfs = sbdep->sb_fs;
9213 fs = sbdep->sb_ump->um_fs;
9214 inodedep = first_unlinked_inodedep(sbdep->sb_ump);
9216 fs->fs_sujfree = inodedep->id_ino;
9217 inodedep->id_state |= UNLINKPREV;
9220 bpfs->fs_sujfree = fs->fs_sujfree;
9224 * After a superblock is written determine whether it must be written again
9225 * due to a changing unlinked list head.
9228 handle_written_sbdep(sbdep, bp)
9229 struct sbdep *sbdep;
9232 struct inodedep *inodedep;
9236 mtx_assert(&lk, MA_OWNED);
9238 mp = UFSTOVFS(sbdep->sb_ump);
9240 * If the superblock doesn't match the in-memory list start over.
9242 inodedep = first_unlinked_inodedep(sbdep->sb_ump);
9243 if ((inodedep && fs->fs_sujfree != inodedep->id_ino) ||
9244 (inodedep == NULL && fs->fs_sujfree != 0)) {
9248 WORKITEM_FREE(sbdep, D_SBDEP);
9249 if (fs->fs_sujfree == 0)
9252 * Now that we have a record of this inode in stable store allow it
9253 * to be written to free up pending work. Inodes may see a lot of
9254 * write activity after they are unlinked which we must not hold up.
9256 for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) {
9257 if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS)
9258 panic("handle_written_sbdep: Bad inodedep %p (0x%X)",
9259 inodedep, inodedep->id_state);
9260 if (inodedep->id_state & UNLINKONLIST)
9262 inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST;
9269 * Mark an inodedep as unlinked and insert it into the in-memory unlinked list.
9272 unlinked_inodedep(mp, inodedep)
9274 struct inodedep *inodedep;
9276 struct ufsmount *ump;
9278 mtx_assert(&lk, MA_OWNED);
9279 if (MOUNTEDSUJ(mp) == 0)
9282 ump->um_fs->fs_fmod = 1;
9283 if (inodedep->id_state & UNLINKED)
9284 panic("unlinked_inodedep: %p already unlinked\n", inodedep);
9285 inodedep->id_state |= UNLINKED;
9286 TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked);
9290 * Remove an inodedep from the unlinked inodedep list. This may require
9291 * disk writes if the inode has made it that far.
9294 clear_unlinked_inodedep(inodedep)
9295 struct inodedep *inodedep;
9297 struct ufsmount *ump;
9298 struct inodedep *idp;
9299 struct inodedep *idn;
9307 ump = VFSTOUFS(inodedep->id_list.wk_mp);
9309 ino = inodedep->id_ino;
9312 mtx_assert(&lk, MA_OWNED);
9313 KASSERT((inodedep->id_state & UNLINKED) != 0,
9314 ("clear_unlinked_inodedep: inodedep %p not unlinked",
9317 * If nothing has yet been written simply remove us from
9318 * the in memory list and return. This is the most common
9319 * case where handle_workitem_remove() loses the final
9322 if ((inodedep->id_state & UNLINKLINKS) == 0)
9325 * If we have a NEXT pointer and no PREV pointer we can simply
9326 * clear NEXT's PREV and remove ourselves from the list. Be
9327 * careful not to clear PREV if the superblock points at
9330 idn = TAILQ_NEXT(inodedep, id_unlinked);
9331 if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) {
9332 if (idn && fs->fs_sujfree != idn->id_ino)
9333 idn->id_state &= ~UNLINKPREV;
9337 * Here we have an inodedep which is actually linked into
9338 * the list. We must remove it by forcing a write to the
9339 * link before us, whether it be the superblock or an inode.
9340 * Unfortunately the list may change while we're waiting
9341 * on the buf lock for either resource so we must loop until
9342 * we lock the right one. If both the superblock and an
9343 * inode point to this inode we must clear the inode first
9344 * followed by the superblock.
9346 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
9348 if (idp && (idp->id_state & UNLINKNEXT))
9352 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc),
9353 (int)fs->fs_sbsize, 0, 0, 0);
9355 error = bread(ump->um_devvp,
9356 fsbtodb(fs, ino_to_fsba(fs, pino)),
9357 (int)fs->fs_bsize, NOCRED, &bp);
9361 /* If the list has changed restart the loop. */
9362 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
9364 if (idp && (idp->id_state & UNLINKNEXT))
9367 (inodedep->id_state & UNLINKPREV) != UNLINKPREV) {
9374 idn = TAILQ_NEXT(inodedep, id_unlinked);
9378 * Remove us from the in memory list. After this we cannot
9379 * access the inodedep.
9381 KASSERT((inodedep->id_state & UNLINKED) != 0,
9382 ("clear_unlinked_inodedep: inodedep %p not unlinked",
9384 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST);
9385 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked);
9388 * The predecessor's next pointer is manually updated here
9389 * so that the NEXT flag is never cleared for an element
9390 * that is in the list.
9393 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize);
9394 ffs_oldfscompat_write((struct fs *)bp->b_data, ump);
9395 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data,
9397 } else if (fs->fs_magic == FS_UFS1_MAGIC)
9398 ((struct ufs1_dinode *)bp->b_data +
9399 ino_to_fsbo(fs, pino))->di_freelink = nino;
9401 ((struct ufs2_dinode *)bp->b_data +
9402 ino_to_fsbo(fs, pino))->di_freelink = nino;
9404 * If the bwrite fails we have no recourse to recover. The
9405 * filesystem is corrupted already.
9410 * If the superblock pointer still needs to be cleared force
9413 if (fs->fs_sujfree == ino) {
9415 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc),
9416 (int)fs->fs_sbsize, 0, 0, 0);
9417 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize);
9418 ffs_oldfscompat_write((struct fs *)bp->b_data, ump);
9419 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data,
9425 if (fs->fs_sujfree != ino)
9427 panic("clear_unlinked_inodedep: Failed to clear free head");
9429 if (inodedep->id_ino == fs->fs_sujfree)
9430 panic("clear_unlinked_inodedep: Freeing head of free list");
9431 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST);
9432 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked);
9437 * This workitem decrements the inode's link count.
9438 * If the link count reaches zero, the file is removed.
9441 handle_workitem_remove(dirrem, flags)
9442 struct dirrem *dirrem;
9445 struct inodedep *inodedep;
9446 struct workhead dotdotwk;
9447 struct worklist *wk;
9448 struct ufsmount *ump;
9454 if (dirrem->dm_state & ONWORKLIST)
9455 panic("handle_workitem_remove: dirrem %p still on worklist",
9457 oldinum = dirrem->dm_oldinum;
9458 mp = dirrem->dm_list.wk_mp;
9460 flags |= LK_EXCLUSIVE;
9461 if (ffs_vgetf(mp, oldinum, flags, &vp, FFSV_FORCEINSMQ) != 0)
9465 if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0)
9466 panic("handle_workitem_remove: lost inodedep");
9467 if (dirrem->dm_state & ONDEPLIST)
9468 LIST_REMOVE(dirrem, dm_inonext);
9469 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd),
9470 ("handle_workitem_remove: Journal entries not written."));
9473 * Move all dependencies waiting on the remove to complete
9474 * from the dirrem to the inode inowait list to be completed
9475 * after the inode has been updated and written to disk. Any
9476 * marked MKDIR_PARENT are saved to be completed when the .. ref
9479 LIST_INIT(&dotdotwk);
9480 while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) {
9481 WORKLIST_REMOVE(wk);
9482 if (wk->wk_state & MKDIR_PARENT) {
9483 wk->wk_state &= ~MKDIR_PARENT;
9484 WORKLIST_INSERT(&dotdotwk, wk);
9487 WORKLIST_INSERT(&inodedep->id_inowait, wk);
9489 LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list);
9491 * Normal file deletion.
9493 if ((dirrem->dm_state & RMDIR) == 0) {
9495 DIP_SET(ip, i_nlink, ip->i_nlink);
9496 ip->i_flag |= IN_CHANGE;
9497 if (ip->i_nlink < ip->i_effnlink)
9498 panic("handle_workitem_remove: bad file delta");
9499 if (ip->i_nlink == 0)
9500 unlinked_inodedep(mp, inodedep);
9501 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
9502 KASSERT(LIST_EMPTY(&dirrem->dm_jwork),
9503 ("handle_workitem_remove: worklist not empty. %s",
9504 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type)));
9505 WORKITEM_FREE(dirrem, D_DIRREM);
9510 * Directory deletion. Decrement reference count for both the
9511 * just deleted parent directory entry and the reference for ".".
9512 * Arrange to have the reference count on the parent decremented
9513 * to account for the loss of "..".
9516 DIP_SET(ip, i_nlink, ip->i_nlink);
9517 ip->i_flag |= IN_CHANGE;
9518 if (ip->i_nlink < ip->i_effnlink)
9519 panic("handle_workitem_remove: bad dir delta");
9520 if (ip->i_nlink == 0)
9521 unlinked_inodedep(mp, inodedep);
9522 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
9524 * Rename a directory to a new parent. Since, we are both deleting
9525 * and creating a new directory entry, the link count on the new
9526 * directory should not change. Thus we skip the followup dirrem.
9528 if (dirrem->dm_state & DIRCHG) {
9529 KASSERT(LIST_EMPTY(&dirrem->dm_jwork),
9530 ("handle_workitem_remove: DIRCHG and worklist not empty."));
9531 WORKITEM_FREE(dirrem, D_DIRREM);
9535 dirrem->dm_state = ONDEPLIST;
9536 dirrem->dm_oldinum = dirrem->dm_dirinum;
9538 * Place the dirrem on the parent's diremhd list.
9540 if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0)
9541 panic("handle_workitem_remove: lost dir inodedep");
9542 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
9544 * If the allocated inode has never been written to disk, then
9545 * the on-disk inode is zero'ed and we can remove the file
9546 * immediately. When journaling if the inode has been marked
9547 * unlinked and not DEPCOMPLETE we know it can never be written.
9549 inodedep_lookup(mp, oldinum, 0, &inodedep);
9550 if (inodedep == NULL ||
9551 (inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED ||
9552 check_inode_unwritten(inodedep)) {
9555 return handle_workitem_remove(dirrem, flags);
9557 WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list);
9559 ip->i_flag |= IN_CHANGE;
9567 * Inode de-allocation dependencies.
9569 * When an inode's link count is reduced to zero, it can be de-allocated. We
9570 * found it convenient to postpone de-allocation until after the inode is
9571 * written to disk with its new link count (zero). At this point, all of the
9572 * on-disk inode's block pointers are nullified and, with careful dependency
9573 * list ordering, all dependencies related to the inode will be satisfied and
9574 * the corresponding dependency structures de-allocated. So, if/when the
9575 * inode is reused, there will be no mixing of old dependencies with new
9576 * ones. This artificial dependency is set up by the block de-allocation
9577 * procedure above (softdep_setup_freeblocks) and completed by the
9578 * following procedure.
9581 handle_workitem_freefile(freefile)
9582 struct freefile *freefile;
9584 struct workhead wkhd;
9586 struct inodedep *idp;
9587 struct ufsmount *ump;
9590 ump = VFSTOUFS(freefile->fx_list.wk_mp);
9594 error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp);
9597 panic("handle_workitem_freefile: inodedep %p survived", idp);
9600 fs->fs_pendinginodes -= 1;
9603 LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list);
9604 if ((error = ffs_freefile(ump, fs, freefile->fx_devvp,
9605 freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0)
9606 softdep_error("handle_workitem_freefile", error);
9608 WORKITEM_FREE(freefile, D_FREEFILE);
9614 * Helper function which unlinks marker element from work list and returns
9615 * the next element on the list.
9617 static __inline struct worklist *
9618 markernext(struct worklist *marker)
9620 struct worklist *next;
9622 next = LIST_NEXT(marker, wk_list);
9623 LIST_REMOVE(marker, wk_list);
9630 * The dependency structures constructed above are most actively used when file
9631 * system blocks are written to disk. No constraints are placed on when a
9632 * block can be written, but unsatisfied update dependencies are made safe by
9633 * modifying (or replacing) the source memory for the duration of the disk
9634 * write. When the disk write completes, the memory block is again brought
9637 * In-core inode structure reclamation.
9639 * Because there are a finite number of "in-core" inode structures, they are
9640 * reused regularly. By transferring all inode-related dependencies to the
9641 * in-memory inode block and indexing them separately (via "inodedep"s), we
9642 * can allow "in-core" inode structures to be reused at any time and avoid
9643 * any increase in contention.
9645 * Called just before entering the device driver to initiate a new disk I/O.
9646 * The buffer must be locked, thus, no I/O completion operations can occur
9647 * while we are manipulating its associated dependencies.
9650 softdep_disk_io_initiation(bp)
9651 struct buf *bp; /* structure describing disk write to occur */
9653 struct worklist *wk;
9654 struct worklist marker;
9655 struct inodedep *inodedep;
9656 struct freeblks *freeblks;
9657 struct jblkdep *jblkdep;
9658 struct newblk *newblk;
9661 * We only care about write operations. There should never
9662 * be dependencies for reads.
9664 if (bp->b_iocmd != BIO_WRITE)
9665 panic("softdep_disk_io_initiation: not write");
9667 if (bp->b_vflags & BV_BKGRDINPROG)
9668 panic("softdep_disk_io_initiation: Writing buffer with "
9669 "background write in progress: %p", bp);
9671 marker.wk_type = D_LAST + 1; /* Not a normal workitem */
9672 PHOLD(curproc); /* Don't swap out kernel stack */
9676 * Do any necessary pre-I/O processing.
9678 for (wk = LIST_FIRST(&bp->b_dep); wk != NULL;
9679 wk = markernext(&marker)) {
9680 LIST_INSERT_AFTER(wk, &marker, wk_list);
9681 switch (wk->wk_type) {
9684 initiate_write_filepage(WK_PAGEDEP(wk), bp);
9688 inodedep = WK_INODEDEP(wk);
9689 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC)
9690 initiate_write_inodeblock_ufs1(inodedep, bp);
9692 initiate_write_inodeblock_ufs2(inodedep, bp);
9696 initiate_write_indirdep(WK_INDIRDEP(wk), bp);
9700 initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp);
9704 WK_JSEG(wk)->js_buf = NULL;
9708 freeblks = WK_FREEBLKS(wk);
9709 jblkdep = LIST_FIRST(&freeblks->fb_jblkdephd);
9711 * We have to wait for the freeblks to be journaled
9712 * before we can write an inodeblock with updated
9713 * pointers. Be careful to arrange the marker so
9714 * we revisit the freeblks if it's not removed by
9715 * the first jwait().
9717 if (jblkdep != NULL) {
9718 LIST_REMOVE(&marker, wk_list);
9719 LIST_INSERT_BEFORE(wk, &marker, wk_list);
9720 jwait(&jblkdep->jb_list, MNT_WAIT);
9726 * We have to wait for the jnewblk to be journaled
9727 * before we can write to a block if the contents
9728 * may be confused with an earlier file's indirect
9729 * at recovery time. Handle the marker as described
9732 newblk = WK_NEWBLK(wk);
9733 if (newblk->nb_jnewblk != NULL &&
9734 indirblk_lookup(newblk->nb_list.wk_mp,
9735 newblk->nb_newblkno)) {
9736 LIST_REMOVE(&marker, wk_list);
9737 LIST_INSERT_BEFORE(wk, &marker, wk_list);
9738 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
9743 initiate_write_sbdep(WK_SBDEP(wk));
9753 panic("handle_disk_io_initiation: Unexpected type %s",
9754 TYPENAME(wk->wk_type));
9759 PRELE(curproc); /* Allow swapout of kernel stack */
9763 * Called from within the procedure above to deal with unsatisfied
9764 * allocation dependencies in a directory. The buffer must be locked,
9765 * thus, no I/O completion operations can occur while we are
9766 * manipulating its associated dependencies.
9769 initiate_write_filepage(pagedep, bp)
9770 struct pagedep *pagedep;
9773 struct jremref *jremref;
9774 struct jmvref *jmvref;
9775 struct dirrem *dirrem;
9780 if (pagedep->pd_state & IOSTARTED) {
9782 * This can only happen if there is a driver that does not
9783 * understand chaining. Here biodone will reissue the call
9784 * to strategy for the incomplete buffers.
9786 printf("initiate_write_filepage: already started\n");
9789 pagedep->pd_state |= IOSTARTED;
9791 * Wait for all journal remove dependencies to hit the disk.
9792 * We can not allow any potentially conflicting directory adds
9793 * to be visible before removes and rollback is too difficult.
9794 * lk may be dropped and re-acquired, however we hold the buf
9795 * locked so the dependency can not go away.
9797 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next)
9798 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL)
9799 jwait(&jremref->jr_list, MNT_WAIT);
9800 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL)
9801 jwait(&jmvref->jm_list, MNT_WAIT);
9802 for (i = 0; i < DAHASHSZ; i++) {
9803 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
9804 ep = (struct direct *)
9805 ((char *)bp->b_data + dap->da_offset);
9806 if (ep->d_ino != dap->da_newinum)
9807 panic("%s: dir inum %d != new %d",
9808 "initiate_write_filepage",
9809 ep->d_ino, dap->da_newinum);
9810 if (dap->da_state & DIRCHG)
9811 ep->d_ino = dap->da_previous->dm_oldinum;
9814 dap->da_state &= ~ATTACHED;
9815 dap->da_state |= UNDONE;
9821 * Version of initiate_write_inodeblock that handles UFS1 dinodes.
9822 * Note that any bug fixes made to this routine must be done in the
9823 * version found below.
9825 * Called from within the procedure above to deal with unsatisfied
9826 * allocation dependencies in an inodeblock. The buffer must be
9827 * locked, thus, no I/O completion operations can occur while we
9828 * are manipulating its associated dependencies.
9831 initiate_write_inodeblock_ufs1(inodedep, bp)
9832 struct inodedep *inodedep;
9833 struct buf *bp; /* The inode block */
9835 struct allocdirect *adp, *lastadp;
9836 struct ufs1_dinode *dp;
9837 struct ufs1_dinode *sip;
9838 struct inoref *inoref;
9842 ufs_lbn_t prevlbn = 0;
9846 if (inodedep->id_state & IOSTARTED)
9847 panic("initiate_write_inodeblock_ufs1: already started");
9848 inodedep->id_state |= IOSTARTED;
9849 fs = inodedep->id_fs;
9850 dp = (struct ufs1_dinode *)bp->b_data +
9851 ino_to_fsbo(fs, inodedep->id_ino);
9854 * If we're on the unlinked list but have not yet written our
9855 * next pointer initialize it here.
9857 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) {
9858 struct inodedep *inon;
9860 inon = TAILQ_NEXT(inodedep, id_unlinked);
9861 dp->di_freelink = inon ? inon->id_ino : 0;
9864 * If the bitmap is not yet written, then the allocated
9865 * inode cannot be written to disk.
9867 if ((inodedep->id_state & DEPCOMPLETE) == 0) {
9868 if (inodedep->id_savedino1 != NULL)
9869 panic("initiate_write_inodeblock_ufs1: I/O underway");
9871 sip = malloc(sizeof(struct ufs1_dinode),
9872 M_SAVEDINO, M_SOFTDEP_FLAGS);
9874 inodedep->id_savedino1 = sip;
9875 *inodedep->id_savedino1 = *dp;
9876 bzero((caddr_t)dp, sizeof(struct ufs1_dinode));
9877 dp->di_gen = inodedep->id_savedino1->di_gen;
9878 dp->di_freelink = inodedep->id_savedino1->di_freelink;
9882 * If no dependencies, then there is nothing to roll back.
9884 inodedep->id_savedsize = dp->di_size;
9885 inodedep->id_savedextsize = 0;
9886 inodedep->id_savednlink = dp->di_nlink;
9887 if (TAILQ_EMPTY(&inodedep->id_inoupdt) &&
9888 TAILQ_EMPTY(&inodedep->id_inoreflst))
9891 * Revert the link count to that of the first unwritten journal entry.
9893 inoref = TAILQ_FIRST(&inodedep->id_inoreflst);
9895 dp->di_nlink = inoref->if_nlink;
9897 * Set the dependencies to busy.
9899 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
9900 adp = TAILQ_NEXT(adp, ad_next)) {
9902 if (deplist != 0 && prevlbn >= adp->ad_offset)
9903 panic("softdep_write_inodeblock: lbn order");
9904 prevlbn = adp->ad_offset;
9905 if (adp->ad_offset < NDADDR &&
9906 dp->di_db[adp->ad_offset] != adp->ad_newblkno)
9907 panic("%s: direct pointer #%jd mismatch %d != %jd",
9908 "softdep_write_inodeblock",
9909 (intmax_t)adp->ad_offset,
9910 dp->di_db[adp->ad_offset],
9911 (intmax_t)adp->ad_newblkno);
9912 if (adp->ad_offset >= NDADDR &&
9913 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno)
9914 panic("%s: indirect pointer #%jd mismatch %d != %jd",
9915 "softdep_write_inodeblock",
9916 (intmax_t)adp->ad_offset - NDADDR,
9917 dp->di_ib[adp->ad_offset - NDADDR],
9918 (intmax_t)adp->ad_newblkno);
9919 deplist |= 1 << adp->ad_offset;
9920 if ((adp->ad_state & ATTACHED) == 0)
9921 panic("softdep_write_inodeblock: Unknown state 0x%x",
9923 #endif /* INVARIANTS */
9924 adp->ad_state &= ~ATTACHED;
9925 adp->ad_state |= UNDONE;
9928 * The on-disk inode cannot claim to be any larger than the last
9929 * fragment that has been written. Otherwise, the on-disk inode
9930 * might have fragments that were not the last block in the file
9931 * which would corrupt the filesystem.
9933 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
9934 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
9935 if (adp->ad_offset >= NDADDR)
9937 dp->di_db[adp->ad_offset] = adp->ad_oldblkno;
9938 /* keep going until hitting a rollback to a frag */
9939 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
9941 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
9942 for (i = adp->ad_offset + 1; i < NDADDR; i++) {
9944 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
9945 panic("softdep_write_inodeblock: lost dep1");
9946 #endif /* INVARIANTS */
9949 for (i = 0; i < NIADDR; i++) {
9951 if (dp->di_ib[i] != 0 &&
9952 (deplist & ((1 << NDADDR) << i)) == 0)
9953 panic("softdep_write_inodeblock: lost dep2");
9954 #endif /* INVARIANTS */
9960 * If we have zero'ed out the last allocated block of the file,
9961 * roll back the size to the last currently allocated block.
9962 * We know that this last allocated block is a full-sized as
9963 * we already checked for fragments in the loop above.
9965 if (lastadp != NULL &&
9966 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
9967 for (i = lastadp->ad_offset; i >= 0; i--)
9968 if (dp->di_db[i] != 0)
9970 dp->di_size = (i + 1) * fs->fs_bsize;
9973 * The only dependencies are for indirect blocks.
9975 * The file size for indirect block additions is not guaranteed.
9976 * Such a guarantee would be non-trivial to achieve. The conventional
9977 * synchronous write implementation also does not make this guarantee.
9978 * Fsck should catch and fix discrepancies. Arguably, the file size
9979 * can be over-estimated without destroying integrity when the file
9980 * moves into the indirect blocks (i.e., is large). If we want to
9981 * postpone fsck, we are stuck with this argument.
9983 for (; adp; adp = TAILQ_NEXT(adp, ad_next))
9984 dp->di_ib[adp->ad_offset - NDADDR] = 0;
9988 * Version of initiate_write_inodeblock that handles UFS2 dinodes.
9989 * Note that any bug fixes made to this routine must be done in the
9990 * version found above.
9992 * Called from within the procedure above to deal with unsatisfied
9993 * allocation dependencies in an inodeblock. The buffer must be
9994 * locked, thus, no I/O completion operations can occur while we
9995 * are manipulating its associated dependencies.
9998 initiate_write_inodeblock_ufs2(inodedep, bp)
9999 struct inodedep *inodedep;
10000 struct buf *bp; /* The inode block */
10002 struct allocdirect *adp, *lastadp;
10003 struct ufs2_dinode *dp;
10004 struct ufs2_dinode *sip;
10005 struct inoref *inoref;
10009 ufs_lbn_t prevlbn = 0;
10013 if (inodedep->id_state & IOSTARTED)
10014 panic("initiate_write_inodeblock_ufs2: already started");
10015 inodedep->id_state |= IOSTARTED;
10016 fs = inodedep->id_fs;
10017 dp = (struct ufs2_dinode *)bp->b_data +
10018 ino_to_fsbo(fs, inodedep->id_ino);
10021 * If we're on the unlinked list but have not yet written our
10022 * next pointer initialize it here.
10024 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) {
10025 struct inodedep *inon;
10027 inon = TAILQ_NEXT(inodedep, id_unlinked);
10028 dp->di_freelink = inon ? inon->id_ino : 0;
10031 * If the bitmap is not yet written, then the allocated
10032 * inode cannot be written to disk.
10034 if ((inodedep->id_state & DEPCOMPLETE) == 0) {
10035 if (inodedep->id_savedino2 != NULL)
10036 panic("initiate_write_inodeblock_ufs2: I/O underway");
10038 sip = malloc(sizeof(struct ufs2_dinode),
10039 M_SAVEDINO, M_SOFTDEP_FLAGS);
10041 inodedep->id_savedino2 = sip;
10042 *inodedep->id_savedino2 = *dp;
10043 bzero((caddr_t)dp, sizeof(struct ufs2_dinode));
10044 dp->di_gen = inodedep->id_savedino2->di_gen;
10045 dp->di_freelink = inodedep->id_savedino2->di_freelink;
10049 * If no dependencies, then there is nothing to roll back.
10051 inodedep->id_savedsize = dp->di_size;
10052 inodedep->id_savedextsize = dp->di_extsize;
10053 inodedep->id_savednlink = dp->di_nlink;
10054 if (TAILQ_EMPTY(&inodedep->id_inoupdt) &&
10055 TAILQ_EMPTY(&inodedep->id_extupdt) &&
10056 TAILQ_EMPTY(&inodedep->id_inoreflst))
10059 * Revert the link count to that of the first unwritten journal entry.
10061 inoref = TAILQ_FIRST(&inodedep->id_inoreflst);
10063 dp->di_nlink = inoref->if_nlink;
10066 * Set the ext data dependencies to busy.
10068 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
10069 adp = TAILQ_NEXT(adp, ad_next)) {
10071 if (deplist != 0 && prevlbn >= adp->ad_offset)
10072 panic("softdep_write_inodeblock: lbn order");
10073 prevlbn = adp->ad_offset;
10074 if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno)
10075 panic("%s: direct pointer #%jd mismatch %jd != %jd",
10076 "softdep_write_inodeblock",
10077 (intmax_t)adp->ad_offset,
10078 (intmax_t)dp->di_extb[adp->ad_offset],
10079 (intmax_t)adp->ad_newblkno);
10080 deplist |= 1 << adp->ad_offset;
10081 if ((adp->ad_state & ATTACHED) == 0)
10082 panic("softdep_write_inodeblock: Unknown state 0x%x",
10084 #endif /* INVARIANTS */
10085 adp->ad_state &= ~ATTACHED;
10086 adp->ad_state |= UNDONE;
10089 * The on-disk inode cannot claim to be any larger than the last
10090 * fragment that has been written. Otherwise, the on-disk inode
10091 * might have fragments that were not the last block in the ext
10092 * data which would corrupt the filesystem.
10094 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
10095 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
10096 dp->di_extb[adp->ad_offset] = adp->ad_oldblkno;
10097 /* keep going until hitting a rollback to a frag */
10098 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
10100 dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
10101 for (i = adp->ad_offset + 1; i < NXADDR; i++) {
10103 if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0)
10104 panic("softdep_write_inodeblock: lost dep1");
10105 #endif /* INVARIANTS */
10106 dp->di_extb[i] = 0;
10112 * If we have zero'ed out the last allocated block of the ext
10113 * data, roll back the size to the last currently allocated block.
10114 * We know that this last allocated block is a full-sized as
10115 * we already checked for fragments in the loop above.
10117 if (lastadp != NULL &&
10118 dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
10119 for (i = lastadp->ad_offset; i >= 0; i--)
10120 if (dp->di_extb[i] != 0)
10122 dp->di_extsize = (i + 1) * fs->fs_bsize;
10125 * Set the file data dependencies to busy.
10127 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
10128 adp = TAILQ_NEXT(adp, ad_next)) {
10130 if (deplist != 0 && prevlbn >= adp->ad_offset)
10131 panic("softdep_write_inodeblock: lbn order");
10132 if ((adp->ad_state & ATTACHED) == 0)
10133 panic("inodedep %p and adp %p not attached", inodedep, adp);
10134 prevlbn = adp->ad_offset;
10135 if (adp->ad_offset < NDADDR &&
10136 dp->di_db[adp->ad_offset] != adp->ad_newblkno)
10137 panic("%s: direct pointer #%jd mismatch %jd != %jd",
10138 "softdep_write_inodeblock",
10139 (intmax_t)adp->ad_offset,
10140 (intmax_t)dp->di_db[adp->ad_offset],
10141 (intmax_t)adp->ad_newblkno);
10142 if (adp->ad_offset >= NDADDR &&
10143 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno)
10144 panic("%s indirect pointer #%jd mismatch %jd != %jd",
10145 "softdep_write_inodeblock:",
10146 (intmax_t)adp->ad_offset - NDADDR,
10147 (intmax_t)dp->di_ib[adp->ad_offset - NDADDR],
10148 (intmax_t)adp->ad_newblkno);
10149 deplist |= 1 << adp->ad_offset;
10150 if ((adp->ad_state & ATTACHED) == 0)
10151 panic("softdep_write_inodeblock: Unknown state 0x%x",
10153 #endif /* INVARIANTS */
10154 adp->ad_state &= ~ATTACHED;
10155 adp->ad_state |= UNDONE;
10158 * The on-disk inode cannot claim to be any larger than the last
10159 * fragment that has been written. Otherwise, the on-disk inode
10160 * might have fragments that were not the last block in the file
10161 * which would corrupt the filesystem.
10163 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
10164 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
10165 if (adp->ad_offset >= NDADDR)
10167 dp->di_db[adp->ad_offset] = adp->ad_oldblkno;
10168 /* keep going until hitting a rollback to a frag */
10169 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
10171 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
10172 for (i = adp->ad_offset + 1; i < NDADDR; i++) {
10174 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
10175 panic("softdep_write_inodeblock: lost dep2");
10176 #endif /* INVARIANTS */
10179 for (i = 0; i < NIADDR; i++) {
10181 if (dp->di_ib[i] != 0 &&
10182 (deplist & ((1 << NDADDR) << i)) == 0)
10183 panic("softdep_write_inodeblock: lost dep3");
10184 #endif /* INVARIANTS */
10190 * If we have zero'ed out the last allocated block of the file,
10191 * roll back the size to the last currently allocated block.
10192 * We know that this last allocated block is a full-sized as
10193 * we already checked for fragments in the loop above.
10195 if (lastadp != NULL &&
10196 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
10197 for (i = lastadp->ad_offset; i >= 0; i--)
10198 if (dp->di_db[i] != 0)
10200 dp->di_size = (i + 1) * fs->fs_bsize;
10203 * The only dependencies are for indirect blocks.
10205 * The file size for indirect block additions is not guaranteed.
10206 * Such a guarantee would be non-trivial to achieve. The conventional
10207 * synchronous write implementation also does not make this guarantee.
10208 * Fsck should catch and fix discrepancies. Arguably, the file size
10209 * can be over-estimated without destroying integrity when the file
10210 * moves into the indirect blocks (i.e., is large). If we want to
10211 * postpone fsck, we are stuck with this argument.
10213 for (; adp; adp = TAILQ_NEXT(adp, ad_next))
10214 dp->di_ib[adp->ad_offset - NDADDR] = 0;
10218 * Cancel an indirdep as a result of truncation. Release all of the
10219 * children allocindirs and place their journal work on the appropriate
10223 cancel_indirdep(indirdep, bp, freeblks)
10224 struct indirdep *indirdep;
10226 struct freeblks *freeblks;
10228 struct allocindir *aip;
10231 * None of the indirect pointers will ever be visible,
10232 * so they can simply be tossed. GOINGAWAY ensures
10233 * that allocated pointers will be saved in the buffer
10234 * cache until they are freed. Note that they will
10235 * only be able to be found by their physical address
10236 * since the inode mapping the logical address will
10237 * be gone. The save buffer used for the safe copy
10238 * was allocated in setup_allocindir_phase2 using
10239 * the physical address so it could be used for this
10240 * purpose. Hence we swap the safe copy with the real
10241 * copy, allowing the safe copy to be freed and holding
10242 * on to the real copy for later use in indir_trunc.
10244 if (indirdep->ir_state & GOINGAWAY)
10245 panic("cancel_indirdep: already gone");
10246 if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
10247 indirdep->ir_state |= DEPCOMPLETE;
10248 LIST_REMOVE(indirdep, ir_next);
10250 indirdep->ir_state |= GOINGAWAY;
10251 VFSTOUFS(indirdep->ir_list.wk_mp)->um_numindirdeps += 1;
10253 * Pass in bp for blocks still have journal writes
10254 * pending so we can cancel them on their own.
10256 while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != 0)
10257 cancel_allocindir(aip, bp, freeblks, 0);
10258 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0)
10259 cancel_allocindir(aip, NULL, freeblks, 0);
10260 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0)
10261 cancel_allocindir(aip, NULL, freeblks, 0);
10262 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != 0)
10263 cancel_allocindir(aip, NULL, freeblks, 0);
10265 * If there are pending partial truncations we need to keep the
10266 * old block copy around until they complete. This is because
10267 * the current b_data is not a perfect superset of the available
10270 if (TAILQ_EMPTY(&indirdep->ir_trunc))
10271 bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount);
10273 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount);
10274 WORKLIST_REMOVE(&indirdep->ir_list);
10275 WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list);
10276 indirdep->ir_bp = NULL;
10277 indirdep->ir_freeblks = freeblks;
10281 * Free an indirdep once it no longer has new pointers to track.
10284 free_indirdep(indirdep)
10285 struct indirdep *indirdep;
10288 KASSERT(TAILQ_EMPTY(&indirdep->ir_trunc),
10289 ("free_indirdep: Indir trunc list not empty."));
10290 KASSERT(LIST_EMPTY(&indirdep->ir_completehd),
10291 ("free_indirdep: Complete head not empty."));
10292 KASSERT(LIST_EMPTY(&indirdep->ir_writehd),
10293 ("free_indirdep: write head not empty."));
10294 KASSERT(LIST_EMPTY(&indirdep->ir_donehd),
10295 ("free_indirdep: done head not empty."));
10296 KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd),
10297 ("free_indirdep: deplist head not empty."));
10298 KASSERT((indirdep->ir_state & DEPCOMPLETE),
10299 ("free_indirdep: %p still on newblk list.", indirdep));
10300 KASSERT(indirdep->ir_saveddata == NULL,
10301 ("free_indirdep: %p still has saved data.", indirdep));
10302 if (indirdep->ir_state & ONWORKLIST)
10303 WORKLIST_REMOVE(&indirdep->ir_list);
10304 WORKITEM_FREE(indirdep, D_INDIRDEP);
10308 * Called before a write to an indirdep. This routine is responsible for
10309 * rolling back pointers to a safe state which includes only those
10310 * allocindirs which have been completed.
10313 initiate_write_indirdep(indirdep, bp)
10314 struct indirdep *indirdep;
10318 indirdep->ir_state |= IOSTARTED;
10319 if (indirdep->ir_state & GOINGAWAY)
10320 panic("disk_io_initiation: indirdep gone");
10322 * If there are no remaining dependencies, this will be writing
10323 * the real pointers.
10325 if (LIST_EMPTY(&indirdep->ir_deplisthd) &&
10326 TAILQ_EMPTY(&indirdep->ir_trunc))
10329 * Replace up-to-date version with safe version.
10331 if (indirdep->ir_saveddata == NULL) {
10333 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP,
10337 indirdep->ir_state &= ~ATTACHED;
10338 indirdep->ir_state |= UNDONE;
10339 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount);
10340 bcopy(indirdep->ir_savebp->b_data, bp->b_data,
10345 * Called when an inode has been cleared in a cg bitmap. This finally
10346 * eliminates any canceled jaddrefs
10349 softdep_setup_inofree(mp, bp, ino, wkhd)
10353 struct workhead *wkhd;
10355 struct worklist *wk, *wkn;
10356 struct inodedep *inodedep;
10362 fs = VFSTOUFS(mp)->um_fs;
10363 cgp = (struct cg *)bp->b_data;
10364 inosused = cg_inosused(cgp);
10365 if (isset(inosused, ino % fs->fs_ipg))
10366 panic("softdep_setup_inofree: inode %d not freed.", ino);
10367 if (inodedep_lookup(mp, ino, 0, &inodedep))
10368 panic("softdep_setup_inofree: ino %d has existing inodedep %p",
10371 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) {
10372 if (wk->wk_type != D_JADDREF)
10374 WORKLIST_REMOVE(wk);
10376 * We can free immediately even if the jaddref
10377 * isn't attached in a background write as now
10378 * the bitmaps are reconciled.
10380 wk->wk_state |= COMPLETE | ATTACHED;
10381 free_jaddref(WK_JADDREF(wk));
10383 jwork_move(&bp->b_dep, wkhd);
10390 * Called via ffs_blkfree() after a set of frags has been cleared from a cg
10391 * map. Any dependencies waiting for the write to clear are added to the
10392 * buf's list and any jnewblks that are being canceled are discarded
10396 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd)
10399 ufs2_daddr_t blkno;
10401 struct workhead *wkhd;
10403 struct bmsafemap *bmsafemap;
10404 struct jnewblk *jnewblk;
10405 struct worklist *wk;
10410 ufs2_daddr_t jstart;
10418 "softdep_setup_blkfree: blkno %jd frags %d wk head %p",
10419 blkno, frags, wkhd);
10422 /* Lookup the bmsafemap so we track when it is dirty. */
10423 fs = VFSTOUFS(mp)->um_fs;
10424 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL);
10426 * Detach any jnewblks which have been canceled. They must linger
10427 * until the bitmap is cleared again by ffs_blkfree() to prevent
10428 * an unjournaled allocation from hitting the disk.
10431 while ((wk = LIST_FIRST(wkhd)) != NULL) {
10433 "softdep_setup_blkfree: blkno %jd wk type %d",
10434 blkno, wk->wk_type);
10435 WORKLIST_REMOVE(wk);
10436 if (wk->wk_type != D_JNEWBLK) {
10437 WORKLIST_INSERT(&bmsafemap->sm_freehd, wk);
10440 jnewblk = WK_JNEWBLK(wk);
10441 KASSERT(jnewblk->jn_state & GOINGAWAY,
10442 ("softdep_setup_blkfree: jnewblk not canceled."));
10445 * Assert that this block is free in the bitmap
10446 * before we discard the jnewblk.
10448 cgp = (struct cg *)bp->b_data;
10449 blksfree = cg_blksfree(cgp);
10450 bno = dtogd(fs, jnewblk->jn_blkno);
10451 for (i = jnewblk->jn_oldfrags;
10452 i < jnewblk->jn_frags; i++) {
10453 if (isset(blksfree, bno + i))
10455 panic("softdep_setup_blkfree: not free");
10459 * Even if it's not attached we can free immediately
10460 * as the new bitmap is correct.
10462 wk->wk_state |= COMPLETE | ATTACHED;
10463 free_jnewblk(jnewblk);
10469 * Assert that we are not freeing a block which has an outstanding
10470 * allocation dependency.
10472 fs = VFSTOUFS(mp)->um_fs;
10473 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL);
10474 end = blkno + frags;
10475 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) {
10477 * Don't match against blocks that will be freed when the
10478 * background write is done.
10480 if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) ==
10481 (COMPLETE | DEPCOMPLETE))
10483 jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags;
10484 jend = jnewblk->jn_blkno + jnewblk->jn_frags;
10485 if ((blkno >= jstart && blkno < jend) ||
10486 (end > jstart && end <= jend)) {
10487 printf("state 0x%X %jd - %d %d dep %p\n",
10488 jnewblk->jn_state, jnewblk->jn_blkno,
10489 jnewblk->jn_oldfrags, jnewblk->jn_frags,
10491 panic("softdep_setup_blkfree: "
10492 "%jd-%jd(%d) overlaps with %jd-%jd",
10493 blkno, end, frags, jstart, jend);
10501 * Revert a block allocation when the journal record that describes it
10502 * is not yet written.
10505 jnewblk_rollback(jnewblk, fs, cgp, blksfree)
10506 struct jnewblk *jnewblk;
10511 ufs1_daddr_t fragno;
10517 cgbno = dtogd(fs, jnewblk->jn_blkno);
10519 * We have to test which frags need to be rolled back. We may
10520 * be operating on a stale copy when doing background writes.
10522 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++)
10523 if (isclr(blksfree, cgbno + i))
10528 * This is mostly ffs_blkfree() sans some validation and
10529 * superblock updates.
10531 if (frags == fs->fs_frag) {
10532 fragno = fragstoblks(fs, cgbno);
10533 ffs_setblock(fs, blksfree, fragno);
10534 ffs_clusteracct(fs, cgp, fragno, 1);
10535 cgp->cg_cs.cs_nbfree++;
10537 cgbno += jnewblk->jn_oldfrags;
10538 bbase = cgbno - fragnum(fs, cgbno);
10539 /* Decrement the old frags. */
10540 blk = blkmap(fs, blksfree, bbase);
10541 ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
10542 /* Deallocate the fragment */
10543 for (i = 0; i < frags; i++)
10544 setbit(blksfree, cgbno + i);
10545 cgp->cg_cs.cs_nffree += frags;
10546 /* Add back in counts associated with the new frags */
10547 blk = blkmap(fs, blksfree, bbase);
10548 ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
10549 /* If a complete block has been reassembled, account for it. */
10550 fragno = fragstoblks(fs, bbase);
10551 if (ffs_isblock(fs, blksfree, fragno)) {
10552 cgp->cg_cs.cs_nffree -= fs->fs_frag;
10553 ffs_clusteracct(fs, cgp, fragno, 1);
10554 cgp->cg_cs.cs_nbfree++;
10558 jnewblk->jn_state &= ~ATTACHED;
10559 jnewblk->jn_state |= UNDONE;
10565 initiate_write_bmsafemap(bmsafemap, bp)
10566 struct bmsafemap *bmsafemap;
10567 struct buf *bp; /* The cg block. */
10569 struct jaddref *jaddref;
10570 struct jnewblk *jnewblk;
10577 if (bmsafemap->sm_state & IOSTARTED)
10579 bmsafemap->sm_state |= IOSTARTED;
10581 * Clear any inode allocations which are pending journal writes.
10583 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) {
10584 cgp = (struct cg *)bp->b_data;
10585 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
10586 inosused = cg_inosused(cgp);
10587 LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) {
10588 ino = jaddref->ja_ino % fs->fs_ipg;
10589 if (isset(inosused, ino)) {
10590 if ((jaddref->ja_mode & IFMT) == IFDIR)
10591 cgp->cg_cs.cs_ndir--;
10592 cgp->cg_cs.cs_nifree++;
10593 clrbit(inosused, ino);
10594 jaddref->ja_state &= ~ATTACHED;
10595 jaddref->ja_state |= UNDONE;
10598 panic("initiate_write_bmsafemap: inode %d "
10599 "marked free", jaddref->ja_ino);
10603 * Clear any block allocations which are pending journal writes.
10605 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) {
10606 cgp = (struct cg *)bp->b_data;
10607 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
10608 blksfree = cg_blksfree(cgp);
10609 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) {
10610 if (jnewblk_rollback(jnewblk, fs, cgp, blksfree))
10612 panic("initiate_write_bmsafemap: block %jd "
10613 "marked free", jnewblk->jn_blkno);
10617 * Move allocation lists to the written lists so they can be
10618 * cleared once the block write is complete.
10620 LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr,
10621 inodedep, id_deps);
10622 LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr,
10624 LIST_SWAP(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, worklist,
10629 * This routine is called during the completion interrupt
10630 * service routine for a disk write (from the procedure called
10631 * by the device driver to inform the filesystem caches of
10632 * a request completion). It should be called early in this
10633 * procedure, before the block is made available to other
10634 * processes or other routines are called.
10638 softdep_disk_write_complete(bp)
10639 struct buf *bp; /* describes the completed disk write */
10641 struct worklist *wk;
10642 struct worklist *owk;
10643 struct workhead reattach;
10644 struct freeblks *freeblks;
10648 * If an error occurred while doing the write, then the data
10649 * has not hit the disk and the dependencies cannot be unrolled.
10651 if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0)
10653 LIST_INIT(&reattach);
10655 * This lock must not be released anywhere in this code segment.
10660 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) {
10661 WORKLIST_REMOVE(wk);
10662 dep_write[wk->wk_type]++;
10664 panic("duplicate worklist: %p\n", wk);
10666 switch (wk->wk_type) {
10669 if (handle_written_filepage(WK_PAGEDEP(wk), bp))
10670 WORKLIST_INSERT(&reattach, wk);
10674 if (handle_written_inodeblock(WK_INODEDEP(wk), bp))
10675 WORKLIST_INSERT(&reattach, wk);
10679 if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp))
10680 WORKLIST_INSERT(&reattach, wk);
10684 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY);
10687 case D_ALLOCDIRECT:
10688 wk->wk_state |= COMPLETE;
10689 handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL);
10693 wk->wk_state |= COMPLETE;
10694 handle_allocindir_partdone(WK_ALLOCINDIR(wk));
10698 if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp))
10699 WORKLIST_INSERT(&reattach, wk);
10703 wk->wk_state |= COMPLETE;
10704 freeblks = WK_FREEBLKS(wk);
10705 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE &&
10706 LIST_EMPTY(&freeblks->fb_jblkdephd))
10707 add_to_worklist(wk, WK_NODELAY);
10711 handle_written_freework(WK_FREEWORK(wk));
10715 free_jsegdep(WK_JSEGDEP(wk));
10719 handle_written_jseg(WK_JSEG(wk), bp);
10723 if (handle_written_sbdep(WK_SBDEP(wk), bp))
10724 WORKLIST_INSERT(&reattach, wk);
10728 free_freedep(WK_FREEDEP(wk));
10732 panic("handle_disk_write_complete: Unknown type %s",
10733 TYPENAME(wk->wk_type));
10738 * Reattach any requests that must be redone.
10740 while ((wk = LIST_FIRST(&reattach)) != NULL) {
10741 WORKLIST_REMOVE(wk);
10742 WORKLIST_INSERT(&bp->b_dep, wk);
10750 * Called from within softdep_disk_write_complete above. Note that
10751 * this routine is always called from interrupt level with further
10752 * splbio interrupts blocked.
10755 handle_allocdirect_partdone(adp, wkhd)
10756 struct allocdirect *adp; /* the completed allocdirect */
10757 struct workhead *wkhd; /* Work to do when inode is writtne. */
10759 struct allocdirectlst *listhead;
10760 struct allocdirect *listadp;
10761 struct inodedep *inodedep;
10764 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
10767 * The on-disk inode cannot claim to be any larger than the last
10768 * fragment that has been written. Otherwise, the on-disk inode
10769 * might have fragments that were not the last block in the file
10770 * which would corrupt the filesystem. Thus, we cannot free any
10771 * allocdirects after one whose ad_oldblkno claims a fragment as
10772 * these blocks must be rolled back to zero before writing the inode.
10773 * We check the currently active set of allocdirects in id_inoupdt
10774 * or id_extupdt as appropriate.
10776 inodedep = adp->ad_inodedep;
10777 bsize = inodedep->id_fs->fs_bsize;
10778 if (adp->ad_state & EXTDATA)
10779 listhead = &inodedep->id_extupdt;
10781 listhead = &inodedep->id_inoupdt;
10782 TAILQ_FOREACH(listadp, listhead, ad_next) {
10783 /* found our block */
10784 if (listadp == adp)
10786 /* continue if ad_oldlbn is not a fragment */
10787 if (listadp->ad_oldsize == 0 ||
10788 listadp->ad_oldsize == bsize)
10790 /* hit a fragment */
10794 * If we have reached the end of the current list without
10795 * finding the just finished dependency, then it must be
10796 * on the future dependency list. Future dependencies cannot
10797 * be freed until they are moved to the current list.
10799 if (listadp == NULL) {
10801 if (adp->ad_state & EXTDATA)
10802 listhead = &inodedep->id_newextupdt;
10804 listhead = &inodedep->id_newinoupdt;
10805 TAILQ_FOREACH(listadp, listhead, ad_next)
10806 /* found our block */
10807 if (listadp == adp)
10809 if (listadp == NULL)
10810 panic("handle_allocdirect_partdone: lost dep");
10815 * If we have found the just finished dependency, then queue
10816 * it along with anything that follows it that is complete.
10817 * Since the pointer has not yet been written in the inode
10818 * as the dependency prevents it, place the allocdirect on the
10819 * bufwait list where it will be freed once the pointer is
10823 wkhd = &inodedep->id_bufwait;
10824 for (; adp; adp = listadp) {
10825 listadp = TAILQ_NEXT(adp, ad_next);
10826 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
10828 TAILQ_REMOVE(listhead, adp, ad_next);
10829 WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list);
10834 * Called from within softdep_disk_write_complete above. This routine
10835 * completes successfully written allocindirs.
10838 handle_allocindir_partdone(aip)
10839 struct allocindir *aip; /* the completed allocindir */
10841 struct indirdep *indirdep;
10843 if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE)
10845 indirdep = aip->ai_indirdep;
10846 LIST_REMOVE(aip, ai_next);
10848 * Don't set a pointer while the buffer is undergoing IO or while
10849 * we have active truncations.
10851 if (indirdep->ir_state & UNDONE || !TAILQ_EMPTY(&indirdep->ir_trunc)) {
10852 LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next);
10855 if (indirdep->ir_state & UFS1FMT)
10856 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
10859 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
10862 * Await the pointer write before freeing the allocindir.
10864 LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next);
10868 * Release segments held on a jwork list.
10872 struct workhead *wkhd;
10874 struct worklist *wk;
10876 while ((wk = LIST_FIRST(wkhd)) != NULL) {
10877 WORKLIST_REMOVE(wk);
10878 switch (wk->wk_type) {
10880 free_jsegdep(WK_JSEGDEP(wk));
10883 free_freedep(WK_FREEDEP(wk));
10886 rele_jseg(WK_JSEG(WK_FREEFRAG(wk)->ff_jdep));
10887 WORKITEM_FREE(wk, D_FREEFRAG);
10890 handle_written_freework(WK_FREEWORK(wk));
10893 panic("handle_jwork: Unknown type %s\n",
10894 TYPENAME(wk->wk_type));
10900 * Handle the bufwait list on an inode when it is safe to release items
10901 * held there. This normally happens after an inode block is written but
10902 * may be delayed and handled later if there are pending journal items that
10903 * are not yet safe to be released.
10905 static struct freefile *
10906 handle_bufwait(inodedep, refhd)
10907 struct inodedep *inodedep;
10908 struct workhead *refhd;
10910 struct jaddref *jaddref;
10911 struct freefile *freefile;
10912 struct worklist *wk;
10915 while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) {
10916 WORKLIST_REMOVE(wk);
10917 switch (wk->wk_type) {
10920 * We defer adding freefile to the worklist
10921 * until all other additions have been made to
10922 * ensure that it will be done after all the
10923 * old blocks have been freed.
10925 if (freefile != NULL)
10926 panic("handle_bufwait: freefile");
10927 freefile = WK_FREEFILE(wk);
10931 handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT);
10935 diradd_inode_written(WK_DIRADD(wk), inodedep);
10939 wk->wk_state |= COMPLETE;
10940 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE)
10941 add_to_worklist(wk, 0);
10945 wk->wk_state |= COMPLETE;
10946 add_to_worklist(wk, 0);
10949 case D_ALLOCDIRECT:
10951 free_newblk(WK_NEWBLK(wk));
10955 wk->wk_state |= COMPLETE;
10956 free_jnewblk(WK_JNEWBLK(wk));
10960 * Save freed journal segments and add references on
10961 * the supplied list which will delay their release
10962 * until the cg bitmap is cleared on disk.
10966 free_jsegdep(WK_JSEGDEP(wk));
10968 WORKLIST_INSERT(refhd, wk);
10972 jaddref = WK_JADDREF(wk);
10973 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref,
10976 * Transfer any jaddrefs to the list to be freed with
10977 * the bitmap if we're handling a removed file.
10979 if (refhd == NULL) {
10980 wk->wk_state |= COMPLETE;
10981 free_jaddref(jaddref);
10983 WORKLIST_INSERT(refhd, wk);
10987 panic("handle_bufwait: Unknown type %p(%s)",
10988 wk, TYPENAME(wk->wk_type));
10995 * Called from within softdep_disk_write_complete above to restore
10996 * in-memory inode block contents to their most up-to-date state. Note
10997 * that this routine is always called from interrupt level with further
10998 * splbio interrupts blocked.
11001 handle_written_inodeblock(inodedep, bp)
11002 struct inodedep *inodedep;
11003 struct buf *bp; /* buffer containing the inode block */
11005 struct freefile *freefile;
11006 struct allocdirect *adp, *nextadp;
11007 struct ufs1_dinode *dp1 = NULL;
11008 struct ufs2_dinode *dp2 = NULL;
11009 struct workhead wkhd;
11010 int hadchanges, fstype;
11016 if ((inodedep->id_state & IOSTARTED) == 0)
11017 panic("handle_written_inodeblock: not started");
11018 inodedep->id_state &= ~IOSTARTED;
11019 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) {
11021 dp1 = (struct ufs1_dinode *)bp->b_data +
11022 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
11023 freelink = dp1->di_freelink;
11026 dp2 = (struct ufs2_dinode *)bp->b_data +
11027 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
11028 freelink = dp2->di_freelink;
11031 * Leave this inodeblock dirty until it's in the list.
11033 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED) {
11034 struct inodedep *inon;
11036 inon = TAILQ_NEXT(inodedep, id_unlinked);
11037 if ((inon == NULL && freelink == 0) ||
11038 (inon && inon->id_ino == freelink)) {
11040 inon->id_state |= UNLINKPREV;
11041 inodedep->id_state |= UNLINKNEXT;
11046 * If we had to rollback the inode allocation because of
11047 * bitmaps being incomplete, then simply restore it.
11048 * Keep the block dirty so that it will not be reclaimed until
11049 * all associated dependencies have been cleared and the
11050 * corresponding updates written to disk.
11052 if (inodedep->id_savedino1 != NULL) {
11054 if (fstype == UFS1)
11055 *dp1 = *inodedep->id_savedino1;
11057 *dp2 = *inodedep->id_savedino2;
11058 free(inodedep->id_savedino1, M_SAVEDINO);
11059 inodedep->id_savedino1 = NULL;
11060 if ((bp->b_flags & B_DELWRI) == 0)
11061 stat_inode_bitmap++;
11064 * If the inode is clear here and GOINGAWAY it will never
11065 * be written. Process the bufwait and clear any pending
11066 * work which may include the freefile.
11068 if (inodedep->id_state & GOINGAWAY)
11072 inodedep->id_state |= COMPLETE;
11074 * Roll forward anything that had to be rolled back before
11075 * the inode could be updated.
11077 for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) {
11078 nextadp = TAILQ_NEXT(adp, ad_next);
11079 if (adp->ad_state & ATTACHED)
11080 panic("handle_written_inodeblock: new entry");
11081 if (fstype == UFS1) {
11082 if (adp->ad_offset < NDADDR) {
11083 if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno)
11084 panic("%s %s #%jd mismatch %d != %jd",
11085 "handle_written_inodeblock:",
11087 (intmax_t)adp->ad_offset,
11088 dp1->di_db[adp->ad_offset],
11089 (intmax_t)adp->ad_oldblkno);
11090 dp1->di_db[adp->ad_offset] = adp->ad_newblkno;
11092 if (dp1->di_ib[adp->ad_offset - NDADDR] != 0)
11093 panic("%s: %s #%jd allocated as %d",
11094 "handle_written_inodeblock",
11095 "indirect pointer",
11096 (intmax_t)adp->ad_offset - NDADDR,
11097 dp1->di_ib[adp->ad_offset - NDADDR]);
11098 dp1->di_ib[adp->ad_offset - NDADDR] =
11102 if (adp->ad_offset < NDADDR) {
11103 if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno)
11104 panic("%s: %s #%jd %s %jd != %jd",
11105 "handle_written_inodeblock",
11107 (intmax_t)adp->ad_offset, "mismatch",
11108 (intmax_t)dp2->di_db[adp->ad_offset],
11109 (intmax_t)adp->ad_oldblkno);
11110 dp2->di_db[adp->ad_offset] = adp->ad_newblkno;
11112 if (dp2->di_ib[adp->ad_offset - NDADDR] != 0)
11113 panic("%s: %s #%jd allocated as %jd",
11114 "handle_written_inodeblock",
11115 "indirect pointer",
11116 (intmax_t)adp->ad_offset - NDADDR,
11118 dp2->di_ib[adp->ad_offset - NDADDR]);
11119 dp2->di_ib[adp->ad_offset - NDADDR] =
11123 adp->ad_state &= ~UNDONE;
11124 adp->ad_state |= ATTACHED;
11127 for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) {
11128 nextadp = TAILQ_NEXT(adp, ad_next);
11129 if (adp->ad_state & ATTACHED)
11130 panic("handle_written_inodeblock: new entry");
11131 if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno)
11132 panic("%s: direct pointers #%jd %s %jd != %jd",
11133 "handle_written_inodeblock",
11134 (intmax_t)adp->ad_offset, "mismatch",
11135 (intmax_t)dp2->di_extb[adp->ad_offset],
11136 (intmax_t)adp->ad_oldblkno);
11137 dp2->di_extb[adp->ad_offset] = adp->ad_newblkno;
11138 adp->ad_state &= ~UNDONE;
11139 adp->ad_state |= ATTACHED;
11142 if (hadchanges && (bp->b_flags & B_DELWRI) == 0)
11143 stat_direct_blk_ptrs++;
11145 * Reset the file size to its most up-to-date value.
11147 if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1)
11148 panic("handle_written_inodeblock: bad size");
11149 if (inodedep->id_savednlink > LINK_MAX)
11150 panic("handle_written_inodeblock: Invalid link count "
11151 "%d for inodedep %p", inodedep->id_savednlink, inodedep);
11152 if (fstype == UFS1) {
11153 if (dp1->di_nlink != inodedep->id_savednlink) {
11154 dp1->di_nlink = inodedep->id_savednlink;
11157 if (dp1->di_size != inodedep->id_savedsize) {
11158 dp1->di_size = inodedep->id_savedsize;
11162 if (dp2->di_nlink != inodedep->id_savednlink) {
11163 dp2->di_nlink = inodedep->id_savednlink;
11166 if (dp2->di_size != inodedep->id_savedsize) {
11167 dp2->di_size = inodedep->id_savedsize;
11170 if (dp2->di_extsize != inodedep->id_savedextsize) {
11171 dp2->di_extsize = inodedep->id_savedextsize;
11175 inodedep->id_savedsize = -1;
11176 inodedep->id_savedextsize = -1;
11177 inodedep->id_savednlink = -1;
11179 * If there were any rollbacks in the inode block, then it must be
11180 * marked dirty so that its will eventually get written back in
11181 * its correct form.
11187 * Process any allocdirects that completed during the update.
11189 if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL)
11190 handle_allocdirect_partdone(adp, &wkhd);
11191 if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL)
11192 handle_allocdirect_partdone(adp, &wkhd);
11194 * Process deallocations that were held pending until the
11195 * inode had been written to disk. Freeing of the inode
11196 * is delayed until after all blocks have been freed to
11197 * avoid creation of new <vfsid, inum, lbn> triples
11198 * before the old ones have been deleted. Completely
11199 * unlinked inodes are not processed until the unlinked
11200 * inode list is written or the last reference is removed.
11202 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) {
11203 freefile = handle_bufwait(inodedep, NULL);
11204 if (freefile && !LIST_EMPTY(&wkhd)) {
11205 WORKLIST_INSERT(&wkhd, &freefile->fx_list);
11210 * Move rolled forward dependency completions to the bufwait list
11211 * now that those that were already written have been processed.
11213 if (!LIST_EMPTY(&wkhd) && hadchanges == 0)
11214 panic("handle_written_inodeblock: bufwait but no changes");
11215 jwork_move(&inodedep->id_bufwait, &wkhd);
11217 if (freefile != NULL) {
11219 * If the inode is goingaway it was never written. Fake up
11220 * the state here so free_inodedep() can succeed.
11222 if (inodedep->id_state & GOINGAWAY)
11223 inodedep->id_state |= COMPLETE | DEPCOMPLETE;
11224 if (free_inodedep(inodedep) == 0)
11225 panic("handle_written_inodeblock: live inodedep %p",
11227 add_to_worklist(&freefile->fx_list, 0);
11232 * If no outstanding dependencies, free it.
11234 if (free_inodedep(inodedep) ||
11235 (TAILQ_FIRST(&inodedep->id_inoreflst) == 0 &&
11236 TAILQ_FIRST(&inodedep->id_inoupdt) == 0 &&
11237 TAILQ_FIRST(&inodedep->id_extupdt) == 0 &&
11238 LIST_FIRST(&inodedep->id_bufwait) == 0))
11240 return (hadchanges);
11244 handle_written_indirdep(indirdep, bp, bpp)
11245 struct indirdep *indirdep;
11249 struct allocindir *aip;
11253 if (indirdep->ir_state & GOINGAWAY)
11254 panic("handle_written_indirdep: indirdep gone");
11255 if ((indirdep->ir_state & IOSTARTED) == 0)
11256 panic("handle_written_indirdep: IO not started");
11259 * If there were rollbacks revert them here.
11261 if (indirdep->ir_saveddata) {
11262 bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount);
11263 if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
11264 free(indirdep->ir_saveddata, M_INDIRDEP);
11265 indirdep->ir_saveddata = NULL;
11269 indirdep->ir_state &= ~(UNDONE | IOSTARTED);
11270 indirdep->ir_state |= ATTACHED;
11272 * Move allocindirs with written pointers to the completehd if
11273 * the indirdep's pointer is not yet written. Otherwise
11276 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0) {
11277 LIST_REMOVE(aip, ai_next);
11278 if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
11279 LIST_INSERT_HEAD(&indirdep->ir_completehd, aip,
11281 newblk_freefrag(&aip->ai_block);
11284 free_newblk(&aip->ai_block);
11287 * Move allocindirs that have finished dependency processing from
11288 * the done list to the write list after updating the pointers.
11290 if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
11291 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) {
11292 handle_allocindir_partdone(aip);
11293 if (aip == LIST_FIRST(&indirdep->ir_donehd))
11294 panic("disk_write_complete: not gone");
11299 * Preserve the indirdep if there were any changes or if it is not
11300 * yet valid on disk.
11303 stat_indir_blk_ptrs++;
11308 * If there were no changes we can discard the savedbp and detach
11309 * ourselves from the buf. We are only carrying completed pointers
11312 sbp = indirdep->ir_savebp;
11313 sbp->b_flags |= B_INVAL | B_NOCACHE;
11314 indirdep->ir_savebp = NULL;
11315 indirdep->ir_bp = NULL;
11317 panic("handle_written_indirdep: bp already exists.");
11320 * The indirdep may not be freed until its parent points at it.
11322 if (indirdep->ir_state & DEPCOMPLETE)
11323 free_indirdep(indirdep);
11329 * Process a diradd entry after its dependent inode has been written.
11330 * This routine must be called with splbio interrupts blocked.
11333 diradd_inode_written(dap, inodedep)
11334 struct diradd *dap;
11335 struct inodedep *inodedep;
11338 dap->da_state |= COMPLETE;
11339 complete_diradd(dap);
11340 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
11344 * Returns true if the bmsafemap will have rollbacks when written. Must
11345 * only be called with lk and the buf lock on the cg held.
11348 bmsafemap_backgroundwrite(bmsafemap, bp)
11349 struct bmsafemap *bmsafemap;
11354 dirty = !LIST_EMPTY(&bmsafemap->sm_jaddrefhd) |
11355 !LIST_EMPTY(&bmsafemap->sm_jnewblkhd);
11357 * If we're initiating a background write we need to process the
11358 * rollbacks as they exist now, not as they exist when IO starts.
11359 * No other consumers will look at the contents of the shadowed
11360 * buf so this is safe to do here.
11362 if (bp->b_xflags & BX_BKGRDMARKER)
11363 initiate_write_bmsafemap(bmsafemap, bp);
11369 * Re-apply an allocation when a cg write is complete.
11372 jnewblk_rollforward(jnewblk, fs, cgp, blksfree)
11373 struct jnewblk *jnewblk;
11378 ufs1_daddr_t fragno;
11379 ufs2_daddr_t blkno;
11385 cgbno = dtogd(fs, jnewblk->jn_blkno);
11386 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) {
11387 if (isclr(blksfree, cgbno + i))
11388 panic("jnewblk_rollforward: re-allocated fragment");
11391 if (frags == fs->fs_frag) {
11392 blkno = fragstoblks(fs, cgbno);
11393 ffs_clrblock(fs, blksfree, (long)blkno);
11394 ffs_clusteracct(fs, cgp, blkno, -1);
11395 cgp->cg_cs.cs_nbfree--;
11397 bbase = cgbno - fragnum(fs, cgbno);
11398 cgbno += jnewblk->jn_oldfrags;
11399 /* If a complete block had been reassembled, account for it. */
11400 fragno = fragstoblks(fs, bbase);
11401 if (ffs_isblock(fs, blksfree, fragno)) {
11402 cgp->cg_cs.cs_nffree += fs->fs_frag;
11403 ffs_clusteracct(fs, cgp, fragno, -1);
11404 cgp->cg_cs.cs_nbfree--;
11406 /* Decrement the old frags. */
11407 blk = blkmap(fs, blksfree, bbase);
11408 ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
11409 /* Allocate the fragment */
11410 for (i = 0; i < frags; i++)
11411 clrbit(blksfree, cgbno + i);
11412 cgp->cg_cs.cs_nffree -= frags;
11413 /* Add back in counts associated with the new frags */
11414 blk = blkmap(fs, blksfree, bbase);
11415 ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
11421 * Complete a write to a bmsafemap structure. Roll forward any bitmap
11422 * changes if it's not a background write. Set all written dependencies
11423 * to DEPCOMPLETE and free the structure if possible.
11426 handle_written_bmsafemap(bmsafemap, bp)
11427 struct bmsafemap *bmsafemap;
11430 struct newblk *newblk;
11431 struct inodedep *inodedep;
11432 struct jaddref *jaddref, *jatmp;
11433 struct jnewblk *jnewblk, *jntmp;
11434 struct ufsmount *ump;
11442 if ((bmsafemap->sm_state & IOSTARTED) == 0)
11443 panic("initiate_write_bmsafemap: Not started\n");
11444 ump = VFSTOUFS(bmsafemap->sm_list.wk_mp);
11446 bmsafemap->sm_state &= ~IOSTARTED;
11448 * Release journal work that was waiting on the write.
11450 handle_jwork(&bmsafemap->sm_freewr);
11453 * Restore unwritten inode allocation pending jaddref writes.
11455 if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) {
11456 cgp = (struct cg *)bp->b_data;
11457 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
11458 inosused = cg_inosused(cgp);
11459 LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd,
11460 ja_bmdeps, jatmp) {
11461 if ((jaddref->ja_state & UNDONE) == 0)
11463 ino = jaddref->ja_ino % fs->fs_ipg;
11464 if (isset(inosused, ino))
11465 panic("handle_written_bmsafemap: "
11466 "re-allocated inode");
11467 if ((bp->b_xflags & BX_BKGRDMARKER) == 0) {
11468 if ((jaddref->ja_mode & IFMT) == IFDIR)
11469 cgp->cg_cs.cs_ndir++;
11470 cgp->cg_cs.cs_nifree--;
11471 setbit(inosused, ino);
11474 jaddref->ja_state &= ~UNDONE;
11475 jaddref->ja_state |= ATTACHED;
11476 free_jaddref(jaddref);
11480 * Restore any block allocations which are pending journal writes.
11482 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) {
11483 cgp = (struct cg *)bp->b_data;
11484 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
11485 blksfree = cg_blksfree(cgp);
11486 LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps,
11488 if ((jnewblk->jn_state & UNDONE) == 0)
11490 if ((bp->b_xflags & BX_BKGRDMARKER) == 0 &&
11491 jnewblk_rollforward(jnewblk, fs, cgp, blksfree))
11493 jnewblk->jn_state &= ~(UNDONE | NEWBLOCK);
11494 jnewblk->jn_state |= ATTACHED;
11495 free_jnewblk(jnewblk);
11498 while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) {
11499 newblk->nb_state |= DEPCOMPLETE;
11500 newblk->nb_state &= ~ONDEPLIST;
11501 newblk->nb_bmsafemap = NULL;
11502 LIST_REMOVE(newblk, nb_deps);
11503 if (newblk->nb_list.wk_type == D_ALLOCDIRECT)
11504 handle_allocdirect_partdone(
11505 WK_ALLOCDIRECT(&newblk->nb_list), NULL);
11506 else if (newblk->nb_list.wk_type == D_ALLOCINDIR)
11507 handle_allocindir_partdone(
11508 WK_ALLOCINDIR(&newblk->nb_list));
11509 else if (newblk->nb_list.wk_type != D_NEWBLK)
11510 panic("handle_written_bmsafemap: Unexpected type: %s",
11511 TYPENAME(newblk->nb_list.wk_type));
11513 while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) {
11514 inodedep->id_state |= DEPCOMPLETE;
11515 inodedep->id_state &= ~ONDEPLIST;
11516 LIST_REMOVE(inodedep, id_deps);
11517 inodedep->id_bmsafemap = NULL;
11519 LIST_REMOVE(bmsafemap, sm_next);
11520 if (chgs == 0 && LIST_EMPTY(&bmsafemap->sm_jaddrefhd) &&
11521 LIST_EMPTY(&bmsafemap->sm_jnewblkhd) &&
11522 LIST_EMPTY(&bmsafemap->sm_newblkhd) &&
11523 LIST_EMPTY(&bmsafemap->sm_inodedephd) &&
11524 LIST_EMPTY(&bmsafemap->sm_freehd)) {
11525 LIST_REMOVE(bmsafemap, sm_hash);
11526 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP);
11529 LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next);
11535 * Try to free a mkdir dependency.
11538 complete_mkdir(mkdir)
11539 struct mkdir *mkdir;
11541 struct diradd *dap;
11543 if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE)
11545 LIST_REMOVE(mkdir, md_mkdirs);
11546 dap = mkdir->md_diradd;
11547 dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY));
11548 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) {
11549 dap->da_state |= DEPCOMPLETE;
11550 complete_diradd(dap);
11552 WORKITEM_FREE(mkdir, D_MKDIR);
11556 * Handle the completion of a mkdir dependency.
11559 handle_written_mkdir(mkdir, type)
11560 struct mkdir *mkdir;
11564 if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type)
11565 panic("handle_written_mkdir: bad type");
11566 mkdir->md_state |= COMPLETE;
11567 complete_mkdir(mkdir);
11571 free_pagedep(pagedep)
11572 struct pagedep *pagedep;
11576 if (pagedep->pd_state & NEWBLOCK)
11578 if (!LIST_EMPTY(&pagedep->pd_dirremhd))
11580 for (i = 0; i < DAHASHSZ; i++)
11581 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i]))
11583 if (!LIST_EMPTY(&pagedep->pd_pendinghd))
11585 if (!LIST_EMPTY(&pagedep->pd_jmvrefhd))
11587 if (pagedep->pd_state & ONWORKLIST)
11588 WORKLIST_REMOVE(&pagedep->pd_list);
11589 LIST_REMOVE(pagedep, pd_hash);
11590 WORKITEM_FREE(pagedep, D_PAGEDEP);
11596 * Called from within softdep_disk_write_complete above.
11597 * A write operation was just completed. Removed inodes can
11598 * now be freed and associated block pointers may be committed.
11599 * Note that this routine is always called from interrupt level
11600 * with further splbio interrupts blocked.
11603 handle_written_filepage(pagedep, bp)
11604 struct pagedep *pagedep;
11605 struct buf *bp; /* buffer containing the written page */
11607 struct dirrem *dirrem;
11608 struct diradd *dap, *nextdap;
11612 if ((pagedep->pd_state & IOSTARTED) == 0)
11613 panic("handle_written_filepage: not started");
11614 pagedep->pd_state &= ~IOSTARTED;
11616 * Process any directory removals that have been committed.
11618 while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) {
11619 LIST_REMOVE(dirrem, dm_next);
11620 dirrem->dm_state |= COMPLETE;
11621 dirrem->dm_dirinum = pagedep->pd_ino;
11622 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd),
11623 ("handle_written_filepage: Journal entries not written."));
11624 add_to_worklist(&dirrem->dm_list, 0);
11627 * Free any directory additions that have been committed.
11628 * If it is a newly allocated block, we have to wait until
11629 * the on-disk directory inode claims the new block.
11631 if ((pagedep->pd_state & NEWBLOCK) == 0)
11632 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
11633 free_diradd(dap, NULL);
11635 * Uncommitted directory entries must be restored.
11637 for (chgs = 0, i = 0; i < DAHASHSZ; i++) {
11638 for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap;
11640 nextdap = LIST_NEXT(dap, da_pdlist);
11641 if (dap->da_state & ATTACHED)
11642 panic("handle_written_filepage: attached");
11643 ep = (struct direct *)
11644 ((char *)bp->b_data + dap->da_offset);
11645 ep->d_ino = dap->da_newinum;
11646 dap->da_state &= ~UNDONE;
11647 dap->da_state |= ATTACHED;
11650 * If the inode referenced by the directory has
11651 * been written out, then the dependency can be
11652 * moved to the pending list.
11654 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
11655 LIST_REMOVE(dap, da_pdlist);
11656 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap,
11662 * If there were any rollbacks in the directory, then it must be
11663 * marked dirty so that its will eventually get written back in
11664 * its correct form.
11667 if ((bp->b_flags & B_DELWRI) == 0)
11673 * If we are not waiting for a new directory block to be
11674 * claimed by its inode, then the pagedep will be freed.
11675 * Otherwise it will remain to track any new entries on
11676 * the page in case they are fsync'ed.
11678 free_pagedep(pagedep);
11683 * Writing back in-core inode structures.
11685 * The filesystem only accesses an inode's contents when it occupies an
11686 * "in-core" inode structure. These "in-core" structures are separate from
11687 * the page frames used to cache inode blocks. Only the latter are
11688 * transferred to/from the disk. So, when the updated contents of the
11689 * "in-core" inode structure are copied to the corresponding in-memory inode
11690 * block, the dependencies are also transferred. The following procedure is
11691 * called when copying a dirty "in-core" inode to a cached inode block.
11695 * Called when an inode is loaded from disk. If the effective link count
11696 * differed from the actual link count when it was last flushed, then we
11697 * need to ensure that the correct effective link count is put back.
11700 softdep_load_inodeblock(ip)
11701 struct inode *ip; /* the "in_core" copy of the inode */
11703 struct inodedep *inodedep;
11706 * Check for alternate nlink count.
11708 ip->i_effnlink = ip->i_nlink;
11710 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0,
11715 ip->i_effnlink -= inodedep->id_nlinkdelta;
11720 * This routine is called just before the "in-core" inode
11721 * information is to be copied to the in-memory inode block.
11722 * Recall that an inode block contains several inodes. If
11723 * the force flag is set, then the dependencies will be
11724 * cleared so that the update can always be made. Note that
11725 * the buffer is locked when this routine is called, so we
11726 * will never be in the middle of writing the inode block
11730 softdep_update_inodeblock(ip, bp, waitfor)
11731 struct inode *ip; /* the "in_core" copy of the inode */
11732 struct buf *bp; /* the buffer containing the inode block */
11733 int waitfor; /* nonzero => update must be allowed */
11735 struct inodedep *inodedep;
11736 struct inoref *inoref;
11737 struct worklist *wk;
11743 mp = UFSTOVFS(ip->i_ump);
11746 * Preserve the freelink that is on disk. clear_unlinked_inodedep()
11747 * does not have access to the in-core ip so must write directly into
11748 * the inode block buffer when setting freelink.
11750 if (fs->fs_magic == FS_UFS1_MAGIC)
11751 DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data +
11752 ino_to_fsbo(fs, ip->i_number))->di_freelink);
11754 DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data +
11755 ino_to_fsbo(fs, ip->i_number))->di_freelink);
11757 * If the effective link count is not equal to the actual link
11758 * count, then we must track the difference in an inodedep while
11759 * the inode is (potentially) tossed out of the cache. Otherwise,
11760 * if there is no existing inodedep, then there are no dependencies
11765 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) {
11767 if (ip->i_effnlink != ip->i_nlink)
11768 panic("softdep_update_inodeblock: bad link count");
11771 if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink)
11772 panic("softdep_update_inodeblock: bad delta");
11774 * If we're flushing all dependencies we must also move any waiting
11775 * for journal writes onto the bufwait list prior to I/O.
11778 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
11779 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
11781 jwait(&inoref->if_list, MNT_WAIT);
11787 * Changes have been initiated. Anything depending on these
11788 * changes cannot occur until this inode has been written.
11790 inodedep->id_state &= ~COMPLETE;
11791 if ((inodedep->id_state & ONWORKLIST) == 0)
11792 WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list);
11794 * Any new dependencies associated with the incore inode must
11795 * now be moved to the list associated with the buffer holding
11796 * the in-memory copy of the inode. Once merged process any
11797 * allocdirects that are completed by the merger.
11799 merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt);
11800 if (!TAILQ_EMPTY(&inodedep->id_inoupdt))
11801 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt),
11803 merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt);
11804 if (!TAILQ_EMPTY(&inodedep->id_extupdt))
11805 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt),
11808 * Now that the inode has been pushed into the buffer, the
11809 * operations dependent on the inode being written to disk
11810 * can be moved to the id_bufwait so that they will be
11811 * processed when the buffer I/O completes.
11813 while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) {
11814 WORKLIST_REMOVE(wk);
11815 WORKLIST_INSERT(&inodedep->id_bufwait, wk);
11818 * Newly allocated inodes cannot be written until the bitmap
11819 * that allocates them have been written (indicated by
11820 * DEPCOMPLETE being set in id_state). If we are doing a
11821 * forced sync (e.g., an fsync on a file), we force the bitmap
11822 * to be written so that the update can be done.
11824 if (waitfor == 0) {
11829 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) {
11833 ibp = inodedep->id_bmsafemap->sm_buf;
11834 ibp = getdirtybuf(ibp, &lk, MNT_WAIT);
11837 * If ibp came back as NULL, the dependency could have been
11838 * freed while we slept. Look it up again, and check to see
11839 * that it has completed.
11841 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0)
11847 if ((error = bwrite(ibp)) != 0)
11848 softdep_error("softdep_update_inodeblock: bwrite", error);
11852 * Merge the a new inode dependency list (such as id_newinoupdt) into an
11853 * old inode dependency list (such as id_inoupdt). This routine must be
11854 * called with splbio interrupts blocked.
11857 merge_inode_lists(newlisthead, oldlisthead)
11858 struct allocdirectlst *newlisthead;
11859 struct allocdirectlst *oldlisthead;
11861 struct allocdirect *listadp, *newadp;
11863 newadp = TAILQ_FIRST(newlisthead);
11864 for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) {
11865 if (listadp->ad_offset < newadp->ad_offset) {
11866 listadp = TAILQ_NEXT(listadp, ad_next);
11869 TAILQ_REMOVE(newlisthead, newadp, ad_next);
11870 TAILQ_INSERT_BEFORE(listadp, newadp, ad_next);
11871 if (listadp->ad_offset == newadp->ad_offset) {
11872 allocdirect_merge(oldlisthead, newadp,
11876 newadp = TAILQ_FIRST(newlisthead);
11878 while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) {
11879 TAILQ_REMOVE(newlisthead, newadp, ad_next);
11880 TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next);
11885 * If we are doing an fsync, then we must ensure that any directory
11886 * entries for the inode have been written after the inode gets to disk.
11890 struct vnode *vp; /* the "in_core" copy of the inode */
11892 struct inodedep *inodedep;
11893 struct pagedep *pagedep;
11894 struct inoref *inoref;
11895 struct worklist *wk;
11896 struct diradd *dap;
11902 struct thread *td = curthread;
11903 int error, flushparent, pagedep_new_block;
11912 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) {
11916 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
11917 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
11919 jwait(&inoref->if_list, MNT_WAIT);
11923 if (!LIST_EMPTY(&inodedep->id_inowait) ||
11924 !TAILQ_EMPTY(&inodedep->id_extupdt) ||
11925 !TAILQ_EMPTY(&inodedep->id_newextupdt) ||
11926 !TAILQ_EMPTY(&inodedep->id_inoupdt) ||
11927 !TAILQ_EMPTY(&inodedep->id_newinoupdt))
11928 panic("softdep_fsync: pending ops %p", inodedep);
11929 for (error = 0, flushparent = 0; ; ) {
11930 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL)
11932 if (wk->wk_type != D_DIRADD)
11933 panic("softdep_fsync: Unexpected type %s",
11934 TYPENAME(wk->wk_type));
11935 dap = WK_DIRADD(wk);
11937 * Flush our parent if this directory entry has a MKDIR_PARENT
11938 * dependency or is contained in a newly allocated block.
11940 if (dap->da_state & DIRCHG)
11941 pagedep = dap->da_previous->dm_pagedep;
11943 pagedep = dap->da_pagedep;
11944 parentino = pagedep->pd_ino;
11945 lbn = pagedep->pd_lbn;
11946 if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE)
11947 panic("softdep_fsync: dirty");
11948 if ((dap->da_state & MKDIR_PARENT) ||
11949 (pagedep->pd_state & NEWBLOCK))
11954 * If we are being fsync'ed as part of vgone'ing this vnode,
11955 * then we will not be able to release and recover the
11956 * vnode below, so we just have to give up on writing its
11957 * directory entry out. It will eventually be written, just
11958 * not now, but then the user was not asking to have it
11959 * written, so we are not breaking any promises.
11961 if (vp->v_iflag & VI_DOOMED)
11964 * We prevent deadlock by always fetching inodes from the
11965 * root, moving down the directory tree. Thus, when fetching
11966 * our parent directory, we first try to get the lock. If
11967 * that fails, we must unlock ourselves before requesting
11968 * the lock on our parent. See the comment in ufs_lookup
11969 * for details on possible races.
11972 if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp,
11973 FFSV_FORCEINSMQ)) {
11974 error = vfs_busy(mp, MBF_NOWAIT);
11978 error = vfs_busy(mp, 0);
11979 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
11983 if (vp->v_iflag & VI_DOOMED) {
11989 error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE,
11990 &pvp, FFSV_FORCEINSMQ);
11992 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
11993 if (vp->v_iflag & VI_DOOMED) {
12002 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps
12003 * that are contained in direct blocks will be resolved by
12004 * doing a ffs_update. Pagedeps contained in indirect blocks
12005 * may require a complete sync'ing of the directory. So, we
12006 * try the cheap and fast ffs_update first, and if that fails,
12007 * then we do the slower ffs_syncvnode of the directory.
12012 if ((error = ffs_update(pvp, 1)) != 0) {
12018 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) {
12019 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) {
12020 if (wk->wk_type != D_DIRADD)
12021 panic("softdep_fsync: Unexpected type %s",
12022 TYPENAME(wk->wk_type));
12023 dap = WK_DIRADD(wk);
12024 if (dap->da_state & DIRCHG)
12025 pagedep = dap->da_previous->dm_pagedep;
12027 pagedep = dap->da_pagedep;
12028 pagedep_new_block = pagedep->pd_state & NEWBLOCK;
12031 if (pagedep_new_block && (error =
12032 ffs_syncvnode(pvp, MNT_WAIT, 0))) {
12042 * Flush directory page containing the inode's name.
12044 error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred,
12047 error = bwrite(bp);
12054 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0)
12062 * Flush all the dirty bitmaps associated with the block device
12063 * before flushing the rest of the dirty blocks so as to reduce
12064 * the number of dependencies that will have to be rolled back.
12069 softdep_fsync_mountdev(vp)
12072 struct buf *bp, *nbp;
12073 struct worklist *wk;
12076 if (!vn_isdisk(vp, NULL))
12077 panic("softdep_fsync_mountdev: vnode not a disk");
12078 bo = &vp->v_bufobj;
12082 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
12084 * If it is already scheduled, skip to the next buffer.
12086 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL))
12089 if ((bp->b_flags & B_DELWRI) == 0)
12090 panic("softdep_fsync_mountdev: not dirty");
12092 * We are only interested in bitmaps with outstanding
12095 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL ||
12096 wk->wk_type != D_BMSAFEMAP ||
12097 (bp->b_vflags & BV_BKGRDINPROG)) {
12104 (void) bawrite(bp);
12113 * Sync all cylinder groups that were dirty at the time this function is
12114 * called. Newly dirtied cgs will be inserted before the sintenel. This
12115 * is used to flush freedep activity that may be holding up writes to a
12119 sync_cgs(mp, waitfor)
12123 struct bmsafemap *bmsafemap;
12124 struct bmsafemap *sintenel;
12125 struct ufsmount *ump;
12129 sintenel = malloc(sizeof(*sintenel), M_BMSAFEMAP, M_ZERO | M_WAITOK);
12130 sintenel->sm_cg = -1;
12131 ump = VFSTOUFS(mp);
12134 LIST_INSERT_HEAD(&ump->softdep_dirtycg, sintenel, sm_next);
12135 for (bmsafemap = LIST_NEXT(sintenel, sm_next); bmsafemap != NULL;
12136 bmsafemap = LIST_NEXT(sintenel, sm_next)) {
12137 /* Skip sintenels and cgs with no work to release. */
12138 if (bmsafemap->sm_cg == -1 ||
12139 (LIST_EMPTY(&bmsafemap->sm_freehd) &&
12140 LIST_EMPTY(&bmsafemap->sm_freewr))) {
12141 LIST_REMOVE(sintenel, sm_next);
12142 LIST_INSERT_AFTER(bmsafemap, sintenel, sm_next);
12146 * If we don't get the lock and we're waiting try again, if
12147 * not move on to the next buf and try to sync it.
12149 bp = getdirtybuf(bmsafemap->sm_buf, &lk, waitfor);
12150 if (bp == NULL && waitfor == MNT_WAIT)
12152 LIST_REMOVE(sintenel, sm_next);
12153 LIST_INSERT_AFTER(bmsafemap, sintenel, sm_next);
12157 if (waitfor == MNT_NOWAIT)
12160 error = bwrite(bp);
12165 LIST_REMOVE(sintenel, sm_next);
12167 free(sintenel, M_BMSAFEMAP);
12172 * This routine is called when we are trying to synchronously flush a
12173 * file. This routine must eliminate any filesystem metadata dependencies
12174 * so that the syncing routine can succeed.
12177 softdep_sync_metadata(struct vnode *vp)
12182 * Ensure that any direct block dependencies have been cleared,
12183 * truncations are started, and inode references are journaled.
12187 * Write all journal records to prevent rollbacks on devvp.
12189 if (vp->v_type == VCHR)
12190 softdep_flushjournal(vp->v_mount);
12191 error = flush_inodedep_deps(vp, vp->v_mount, VTOI(vp)->i_number);
12193 * Ensure that all truncates are written so we won't find deps on
12196 process_truncates(vp);
12203 * This routine is called when we are attempting to sync a buf with
12204 * dependencies. If waitfor is MNT_NOWAIT it attempts to schedule any
12205 * other IO it can but returns EBUSY if the buffer is not yet able to
12206 * be written. Dependencies which will not cause rollbacks will always
12210 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor)
12212 struct indirdep *indirdep;
12213 struct pagedep *pagedep;
12214 struct allocindir *aip;
12215 struct newblk *newblk;
12217 struct worklist *wk;
12221 * For VCHR we just don't want to force flush any dependencies that
12222 * will cause rollbacks.
12224 if (vp->v_type == VCHR) {
12225 if (waitfor == MNT_NOWAIT && softdep_count_dependencies(bp, 0))
12231 * As we hold the buffer locked, none of its dependencies
12236 LIST_FOREACH(wk, &bp->b_dep, wk_list) {
12237 switch (wk->wk_type) {
12239 case D_ALLOCDIRECT:
12241 newblk = WK_NEWBLK(wk);
12242 if (newblk->nb_jnewblk != NULL) {
12243 if (waitfor == MNT_NOWAIT) {
12247 jwait(&newblk->nb_jnewblk->jn_list, waitfor);
12250 if (newblk->nb_state & DEPCOMPLETE ||
12251 waitfor == MNT_NOWAIT)
12253 nbp = newblk->nb_bmsafemap->sm_buf;
12254 nbp = getdirtybuf(nbp, &lk, waitfor);
12258 if ((error = bwrite(nbp)) != 0)
12264 indirdep = WK_INDIRDEP(wk);
12265 if (waitfor == MNT_NOWAIT) {
12266 if (!TAILQ_EMPTY(&indirdep->ir_trunc) ||
12267 !LIST_EMPTY(&indirdep->ir_deplisthd)) {
12272 if (!TAILQ_EMPTY(&indirdep->ir_trunc))
12273 panic("softdep_sync_buf: truncation pending.");
12275 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) {
12276 newblk = (struct newblk *)aip;
12277 if (newblk->nb_jnewblk != NULL) {
12278 jwait(&newblk->nb_jnewblk->jn_list,
12282 if (newblk->nb_state & DEPCOMPLETE)
12284 nbp = newblk->nb_bmsafemap->sm_buf;
12285 nbp = getdirtybuf(nbp, &lk, waitfor);
12289 if ((error = bwrite(nbp)) != 0)
12298 * Only flush directory entries in synchronous passes.
12300 if (waitfor != MNT_WAIT) {
12305 * While syncing snapshots, we must allow recursive
12310 * We are trying to sync a directory that may
12311 * have dependencies on both its own metadata
12312 * and/or dependencies on the inodes of any
12313 * recently allocated files. We walk its diradd
12314 * lists pushing out the associated inode.
12316 pagedep = WK_PAGEDEP(wk);
12317 for (i = 0; i < DAHASHSZ; i++) {
12318 if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0)
12320 if ((error = flush_pagedep_deps(vp, wk->wk_mp,
12321 &pagedep->pd_diraddhd[i]))) {
12336 panic("softdep_sync_buf: Unknown type %s",
12337 TYPENAME(wk->wk_type));
12348 * Flush the dependencies associated with an inodedep.
12349 * Called with splbio blocked.
12352 flush_inodedep_deps(vp, mp, ino)
12357 struct inodedep *inodedep;
12358 struct inoref *inoref;
12359 int error, waitfor;
12362 * This work is done in two passes. The first pass grabs most
12363 * of the buffers and begins asynchronously writing them. The
12364 * only way to wait for these asynchronous writes is to sleep
12365 * on the filesystem vnode which may stay busy for a long time
12366 * if the filesystem is active. So, instead, we make a second
12367 * pass over the dependencies blocking on each write. In the
12368 * usual case we will be blocking against a write that we
12369 * initiated, so when it is done the dependency will have been
12370 * resolved. Thus the second pass is expected to end quickly.
12371 * We give a brief window at the top of the loop to allow
12372 * any pending I/O to complete.
12374 for (error = 0, waitfor = MNT_NOWAIT; ; ) {
12380 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0)
12382 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
12383 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
12385 jwait(&inoref->if_list, MNT_WAIT);
12389 if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) ||
12390 flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) ||
12391 flush_deplist(&inodedep->id_extupdt, waitfor, &error) ||
12392 flush_deplist(&inodedep->id_newextupdt, waitfor, &error))
12395 * If pass2, we are done, otherwise do pass 2.
12397 if (waitfor == MNT_WAIT)
12399 waitfor = MNT_WAIT;
12402 * Try freeing inodedep in case all dependencies have been removed.
12404 if (inodedep_lookup(mp, ino, 0, &inodedep) != 0)
12405 (void) free_inodedep(inodedep);
12410 * Flush an inode dependency list.
12411 * Called with splbio blocked.
12414 flush_deplist(listhead, waitfor, errorp)
12415 struct allocdirectlst *listhead;
12419 struct allocdirect *adp;
12420 struct newblk *newblk;
12423 mtx_assert(&lk, MA_OWNED);
12424 TAILQ_FOREACH(adp, listhead, ad_next) {
12425 newblk = (struct newblk *)adp;
12426 if (newblk->nb_jnewblk != NULL) {
12427 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
12430 if (newblk->nb_state & DEPCOMPLETE)
12432 bp = newblk->nb_bmsafemap->sm_buf;
12433 bp = getdirtybuf(bp, &lk, waitfor);
12435 if (waitfor == MNT_NOWAIT)
12440 if (waitfor == MNT_NOWAIT)
12443 *errorp = bwrite(bp);
12451 * Flush dependencies associated with an allocdirect block.
12454 flush_newblk_dep(vp, mp, lbn)
12459 struct newblk *newblk;
12463 ufs2_daddr_t blkno;
12467 bo = &vp->v_bufobj;
12469 blkno = DIP(ip, i_db[lbn]);
12471 panic("flush_newblk_dep: Missing block");
12474 * Loop until all dependencies related to this block are satisfied.
12475 * We must be careful to restart after each sleep in case a write
12476 * completes some part of this process for us.
12479 if (newblk_lookup(mp, blkno, 0, &newblk) == 0) {
12483 if (newblk->nb_list.wk_type != D_ALLOCDIRECT)
12484 panic("flush_newblk_deps: Bad newblk %p", newblk);
12486 * Flush the journal.
12488 if (newblk->nb_jnewblk != NULL) {
12489 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
12493 * Write the bitmap dependency.
12495 if ((newblk->nb_state & DEPCOMPLETE) == 0) {
12496 bp = newblk->nb_bmsafemap->sm_buf;
12497 bp = getdirtybuf(bp, &lk, MNT_WAIT);
12501 error = bwrite(bp);
12508 * Write the buffer.
12512 bp = gbincore(bo, lbn);
12514 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL |
12515 LK_INTERLOCK, BO_MTX(bo));
12516 if (error == ENOLCK) {
12518 continue; /* Slept, retry */
12521 break; /* Failed */
12522 if (bp->b_flags & B_DELWRI) {
12524 error = bwrite(bp);
12532 * We have to wait for the direct pointers to
12533 * point at the newdirblk before the dependency
12536 error = ffs_update(vp, 1);
12545 * Eliminate a pagedep dependency by flushing out all its diradd dependencies.
12546 * Called with splbio blocked.
12549 flush_pagedep_deps(pvp, mp, diraddhdp)
12552 struct diraddhd *diraddhdp;
12554 struct inodedep *inodedep;
12555 struct inoref *inoref;
12556 struct ufsmount *ump;
12557 struct diradd *dap;
12563 ump = VFSTOUFS(mp);
12565 while ((dap = LIST_FIRST(diraddhdp)) != NULL) {
12567 * Flush ourselves if this directory entry
12568 * has a MKDIR_PARENT dependency.
12570 if (dap->da_state & MKDIR_PARENT) {
12572 if ((error = ffs_update(pvp, 1)) != 0)
12576 * If that cleared dependencies, go on to next.
12578 if (dap != LIST_FIRST(diraddhdp))
12580 if (dap->da_state & MKDIR_PARENT)
12581 panic("flush_pagedep_deps: MKDIR_PARENT");
12584 * A newly allocated directory must have its "." and
12585 * ".." entries written out before its name can be
12586 * committed in its parent.
12588 inum = dap->da_newinum;
12589 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0)
12590 panic("flush_pagedep_deps: lost inode1");
12592 * Wait for any pending journal adds to complete so we don't
12593 * cause rollbacks while syncing.
12595 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
12596 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
12598 jwait(&inoref->if_list, MNT_WAIT);
12602 if (dap->da_state & MKDIR_BODY) {
12604 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp,
12607 error = flush_newblk_dep(vp, mp, 0);
12609 * If we still have the dependency we might need to
12610 * update the vnode to sync the new link count to
12613 if (error == 0 && dap == LIST_FIRST(diraddhdp))
12614 error = ffs_update(vp, 1);
12620 * If that cleared dependencies, go on to next.
12622 if (dap != LIST_FIRST(diraddhdp))
12624 if (dap->da_state & MKDIR_BODY) {
12625 inodedep_lookup(UFSTOVFS(ump), inum, 0,
12627 panic("flush_pagedep_deps: MKDIR_BODY "
12628 "inodedep %p dap %p vp %p",
12629 inodedep, dap, vp);
12633 * Flush the inode on which the directory entry depends.
12634 * Having accounted for MKDIR_PARENT and MKDIR_BODY above,
12635 * the only remaining dependency is that the updated inode
12636 * count must get pushed to disk. The inode has already
12637 * been pushed into its inode buffer (via VOP_UPDATE) at
12638 * the time of the reference count change. So we need only
12639 * locate that buffer, ensure that there will be no rollback
12640 * caused by a bitmap dependency, then write the inode buffer.
12643 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0)
12644 panic("flush_pagedep_deps: lost inode");
12646 * If the inode still has bitmap dependencies,
12647 * push them to disk.
12649 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) {
12650 bp = inodedep->id_bmsafemap->sm_buf;
12651 bp = getdirtybuf(bp, &lk, MNT_WAIT);
12655 if ((error = bwrite(bp)) != 0)
12658 if (dap != LIST_FIRST(diraddhdp))
12662 * If the inode is still sitting in a buffer waiting
12663 * to be written or waiting for the link count to be
12664 * adjusted update it here to flush it to disk.
12666 if (dap == LIST_FIRST(diraddhdp)) {
12668 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp,
12671 error = ffs_update(vp, 1);
12678 * If we have failed to get rid of all the dependencies
12679 * then something is seriously wrong.
12681 if (dap == LIST_FIRST(diraddhdp)) {
12682 inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep);
12683 panic("flush_pagedep_deps: failed to flush "
12684 "inodedep %p ino %d dap %p", inodedep, inum, dap);
12693 * A large burst of file addition or deletion activity can drive the
12694 * memory load excessively high. First attempt to slow things down
12695 * using the techniques below. If that fails, this routine requests
12696 * the offending operations to fall back to running synchronously
12697 * until the memory load returns to a reasonable level.
12700 softdep_slowdown(vp)
12703 struct ufsmount *ump;
12705 int max_softdeps_hard;
12710 * Check for journal space if needed.
12712 if (DOINGSUJ(vp)) {
12713 ump = VFSTOUFS(vp->v_mount);
12714 if (journal_space(ump, 0) == 0)
12717 max_softdeps_hard = max_softdeps * 11 / 10;
12718 if (dep_current[D_DIRREM] < max_softdeps_hard / 2 &&
12719 dep_current[D_INODEDEP] < max_softdeps_hard &&
12720 VFSTOUFS(vp->v_mount)->um_numindirdeps < maxindirdeps &&
12721 dep_current[D_FREEBLKS] < max_softdeps_hard && jlow == 0) {
12725 if (VFSTOUFS(vp->v_mount)->um_numindirdeps >= maxindirdeps || jlow)
12727 stat_sync_limit_hit += 1;
12735 * Called by the allocation routines when they are about to fail
12736 * in the hope that we can free up the requested resource (inodes
12739 * First check to see if the work list has anything on it. If it has,
12740 * clean up entries until we successfully free the requested resource.
12741 * Because this process holds inodes locked, we cannot handle any remove
12742 * requests that might block on a locked inode as that could lead to
12743 * deadlock. If the worklist yields none of the requested resource,
12744 * start syncing out vnodes to free up the needed space.
12747 softdep_request_cleanup(fs, vp, cred, resource)
12750 struct ucred *cred;
12753 struct ufsmount *ump;
12755 struct vnode *lvp, *mvp;
12757 ufs2_daddr_t needed;
12761 * If we are being called because of a process doing a
12762 * copy-on-write, then it is not safe to process any
12763 * worklist items as we will recurse into the copyonwrite
12764 * routine. This will result in an incoherent snapshot.
12765 * If the vnode that we hold is a snapshot, we must avoid
12766 * handling other resources that could cause deadlock.
12768 if ((curthread->td_pflags & TDP_COWINPROGRESS) || IS_SNAPSHOT(VTOI(vp)))
12771 if (resource == FLUSH_BLOCKS_WAIT)
12772 stat_cleanup_blkrequests += 1;
12774 stat_cleanup_inorequests += 1;
12777 ump = VFSTOUFS(mp);
12778 mtx_assert(UFS_MTX(ump), MA_OWNED);
12780 error = ffs_update(vp, 1);
12786 * If we are in need of resources, consider pausing for
12787 * tickdelay to give ourselves some breathing room.
12790 process_removes(vp);
12791 process_truncates(vp);
12792 request_cleanup(UFSTOVFS(ump), resource);
12795 * Now clean up at least as many resources as we will need.
12797 * When requested to clean up inodes, the number that are needed
12798 * is set by the number of simultaneous writers (mnt_writeopcount)
12799 * plus a bit of slop (2) in case some more writers show up while
12802 * When requested to free up space, the amount of space that
12803 * we need is enough blocks to allocate a full-sized segment
12804 * (fs_contigsumsize). The number of such segments that will
12805 * be needed is set by the number of simultaneous writers
12806 * (mnt_writeopcount) plus a bit of slop (2) in case some more
12807 * writers show up while we are cleaning.
12809 * Additionally, if we are unpriviledged and allocating space,
12810 * we need to ensure that we clean up enough blocks to get the
12811 * needed number of blocks over the threshhold of the minimum
12812 * number of blocks required to be kept free by the filesystem
12815 if (resource == FLUSH_INODES_WAIT) {
12816 needed = vp->v_mount->mnt_writeopcount + 2;
12817 } else if (resource == FLUSH_BLOCKS_WAIT) {
12818 needed = (vp->v_mount->mnt_writeopcount + 2) *
12819 fs->fs_contigsumsize;
12820 if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE, 0))
12821 needed += fragstoblks(fs,
12822 roundup((fs->fs_dsize * fs->fs_minfree / 100) -
12823 fs->fs_cstotal.cs_nffree, fs->fs_frag));
12826 printf("softdep_request_cleanup: Unknown resource type %d\n",
12830 starttime = time_second;
12832 if ((resource == FLUSH_BLOCKS_WAIT && ump->softdep_on_worklist > 0 &&
12833 fs->fs_cstotal.cs_nbfree <= needed) ||
12834 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 &&
12835 fs->fs_cstotal.cs_nifree <= needed)) {
12837 if (ump->softdep_on_worklist > 0 &&
12838 process_worklist_item(UFSTOVFS(ump),
12839 ump->softdep_on_worklist, LK_NOWAIT) != 0)
12840 stat_worklist_push += 1;
12844 * If we still need resources and there are no more worklist
12845 * entries to process to obtain them, we have to start flushing
12846 * the dirty vnodes to force the release of additional requests
12847 * to the worklist that we can then process to reap addition
12848 * resources. We walk the vnodes associated with the mount point
12849 * until we get the needed worklist requests that we can reap.
12851 if ((resource == FLUSH_BLOCKS_WAIT &&
12852 fs->fs_cstotal.cs_nbfree <= needed) ||
12853 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 &&
12854 fs->fs_cstotal.cs_nifree <= needed)) {
12855 MNT_VNODE_FOREACH_ALL(lvp, mp, mvp) {
12856 if (TAILQ_FIRST(&lvp->v_bufobj.bo_dirty.bv_hd) == 0) {
12860 if (vget(lvp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT,
12863 if (lvp->v_vflag & VV_NOSYNC) { /* unlinked */
12867 (void) ffs_syncvnode(lvp, MNT_NOWAIT, 0);
12870 lvp = ump->um_devvp;
12871 if (vn_lock(lvp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
12872 VOP_FSYNC(lvp, MNT_NOWAIT, curthread);
12873 VOP_UNLOCK(lvp, 0);
12875 if (ump->softdep_on_worklist > 0) {
12876 stat_cleanup_retries += 1;
12879 stat_cleanup_failures += 1;
12881 if (time_second - starttime > stat_cleanup_high_delay)
12882 stat_cleanup_high_delay = time_second - starttime;
12888 * If memory utilization has gotten too high, deliberately slow things
12889 * down and speed up the I/O processing.
12891 extern struct thread *syncertd;
12893 request_cleanup(mp, resource)
12897 struct thread *td = curthread;
12898 struct ufsmount *ump;
12900 mtx_assert(&lk, MA_OWNED);
12902 * We never hold up the filesystem syncer or buf daemon.
12904 if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF))
12906 ump = VFSTOUFS(mp);
12908 * First check to see if the work list has gotten backlogged.
12909 * If it has, co-opt this process to help clean up two entries.
12910 * Because this process may hold inodes locked, we cannot
12911 * handle any remove requests that might block on a locked
12912 * inode as that could lead to deadlock. We set TDP_SOFTDEP
12913 * to avoid recursively processing the worklist.
12915 if (ump->softdep_on_worklist > max_softdeps / 10) {
12916 td->td_pflags |= TDP_SOFTDEP;
12917 process_worklist_item(mp, 2, LK_NOWAIT);
12918 td->td_pflags &= ~TDP_SOFTDEP;
12919 stat_worklist_push += 2;
12923 * Next, we attempt to speed up the syncer process. If that
12924 * is successful, then we allow the process to continue.
12926 if (softdep_speedup() &&
12927 resource != FLUSH_BLOCKS_WAIT &&
12928 resource != FLUSH_INODES_WAIT)
12931 * If we are resource constrained on inode dependencies, try
12932 * flushing some dirty inodes. Otherwise, we are constrained
12933 * by file deletions, so try accelerating flushes of directories
12934 * with removal dependencies. We would like to do the cleanup
12935 * here, but we probably hold an inode locked at this point and
12936 * that might deadlock against one that we try to clean. So,
12937 * the best that we can do is request the syncer daemon to do
12938 * the cleanup for us.
12940 switch (resource) {
12943 case FLUSH_INODES_WAIT:
12944 stat_ino_limit_push += 1;
12945 req_clear_inodedeps += 1;
12946 stat_countp = &stat_ino_limit_hit;
12950 case FLUSH_BLOCKS_WAIT:
12951 stat_blk_limit_push += 1;
12952 req_clear_remove += 1;
12953 stat_countp = &stat_blk_limit_hit;
12957 panic("request_cleanup: unknown type");
12960 * Hopefully the syncer daemon will catch up and awaken us.
12961 * We wait at most tickdelay before proceeding in any case.
12964 if (callout_pending(&softdep_callout) == FALSE)
12965 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2,
12968 msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0);
12974 * Awaken processes pausing in request_cleanup and clear proc_waiting
12975 * to indicate that there is no longer a timer running.
12983 * The callout_ API has acquired mtx and will hold it around this
12987 wakeup_one(&proc_waiting);
12988 if (proc_waiting > 0)
12989 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2,
12994 * Flush out a directory with at least one removal dependency in an effort to
12995 * reduce the number of dirrem, freefile, and freeblks dependency structures.
13001 struct pagedep_hashhead *pagedephd;
13002 struct pagedep *pagedep;
13003 static int next = 0;
13010 mtx_assert(&lk, MA_OWNED);
13012 for (cnt = 0; cnt < pagedep_hash; cnt++) {
13013 pagedephd = &pagedep_hashtbl[next++];
13014 if (next >= pagedep_hash)
13016 LIST_FOREACH(pagedep, pagedephd, pd_hash) {
13017 if (LIST_EMPTY(&pagedep->pd_dirremhd))
13019 mp = pagedep->pd_list.wk_mp;
13020 ino = pagedep->pd_ino;
13021 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
13026 * Let unmount clear deps
13028 error = vfs_busy(mp, MBF_NOWAIT);
13031 error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp,
13035 softdep_error("clear_remove: vget", error);
13038 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0)))
13039 softdep_error("clear_remove: fsync", error);
13040 bo = &vp->v_bufobj;
13046 vn_finished_write(mp);
13054 * Clear out a block of dirty inodes in an effort to reduce
13055 * the number of inodedep dependency structures.
13058 clear_inodedeps(td)
13061 struct inodedep_hashhead *inodedephd;
13062 struct inodedep *inodedep;
13063 static int next = 0;
13068 ino_t firstino, lastino, ino;
13070 mtx_assert(&lk, MA_OWNED);
13072 * Pick a random inode dependency to be cleared.
13073 * We will then gather up all the inodes in its block
13074 * that have dependencies and flush them out.
13076 for (cnt = 0; cnt < inodedep_hash; cnt++) {
13077 inodedephd = &inodedep_hashtbl[next++];
13078 if (next >= inodedep_hash)
13080 if ((inodedep = LIST_FIRST(inodedephd)) != NULL)
13083 if (inodedep == NULL)
13085 fs = inodedep->id_fs;
13086 mp = inodedep->id_list.wk_mp;
13088 * Find the last inode in the block with dependencies.
13090 firstino = inodedep->id_ino & ~(INOPB(fs) - 1);
13091 for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--)
13092 if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0)
13095 * Asynchronously push all but the last inode with dependencies.
13096 * Synchronously push the last inode with dependencies to ensure
13097 * that the inode block gets written to free up the inodedeps.
13099 for (ino = firstino; ino <= lastino; ino++) {
13100 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0)
13102 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
13105 error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */
13107 vn_finished_write(mp);
13111 if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp,
13112 FFSV_FORCEINSMQ)) != 0) {
13113 softdep_error("clear_inodedeps: vget", error);
13115 vn_finished_write(mp);
13120 if (ino == lastino) {
13121 if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)))
13122 softdep_error("clear_inodedeps: fsync1", error);
13124 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0)))
13125 softdep_error("clear_inodedeps: fsync2", error);
13126 BO_LOCK(&vp->v_bufobj);
13128 BO_UNLOCK(&vp->v_bufobj);
13131 vn_finished_write(mp);
13137 softdep_buf_append(bp, wkhd)
13139 struct workhead *wkhd;
13141 struct worklist *wk;
13144 while ((wk = LIST_FIRST(wkhd)) != NULL) {
13145 WORKLIST_REMOVE(wk);
13146 WORKLIST_INSERT(&bp->b_dep, wk);
13153 softdep_inode_append(ip, cred, wkhd)
13155 struct ucred *cred;
13156 struct workhead *wkhd;
13163 error = bread(ip->i_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
13164 (int)fs->fs_bsize, cred, &bp);
13166 softdep_freework(wkhd);
13169 softdep_buf_append(bp, wkhd);
13174 softdep_freework(wkhd)
13175 struct workhead *wkhd;
13179 handle_jwork(wkhd);
13184 * Function to determine if the buffer has outstanding dependencies
13185 * that will cause a roll-back if the buffer is written. If wantcount
13186 * is set, return number of dependencies, otherwise just yes or no.
13189 softdep_count_dependencies(bp, wantcount)
13193 struct worklist *wk;
13194 struct bmsafemap *bmsafemap;
13195 struct freework *freework;
13196 struct inodedep *inodedep;
13197 struct indirdep *indirdep;
13198 struct freeblks *freeblks;
13199 struct allocindir *aip;
13200 struct pagedep *pagedep;
13201 struct dirrem *dirrem;
13202 struct newblk *newblk;
13203 struct mkdir *mkdir;
13204 struct diradd *dap;
13209 LIST_FOREACH(wk, &bp->b_dep, wk_list) {
13210 switch (wk->wk_type) {
13213 inodedep = WK_INODEDEP(wk);
13214 if ((inodedep->id_state & DEPCOMPLETE) == 0) {
13215 /* bitmap allocation dependency */
13220 if (TAILQ_FIRST(&inodedep->id_inoupdt)) {
13221 /* direct block pointer dependency */
13226 if (TAILQ_FIRST(&inodedep->id_extupdt)) {
13227 /* direct block pointer dependency */
13232 if (TAILQ_FIRST(&inodedep->id_inoreflst)) {
13233 /* Add reference dependency. */
13241 indirdep = WK_INDIRDEP(wk);
13243 TAILQ_FOREACH(freework, &indirdep->ir_trunc, fw_next) {
13244 /* indirect truncation dependency */
13250 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) {
13251 /* indirect block pointer dependency */
13259 pagedep = WK_PAGEDEP(wk);
13260 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) {
13261 if (LIST_FIRST(&dirrem->dm_jremrefhd)) {
13262 /* Journal remove ref dependency. */
13268 for (i = 0; i < DAHASHSZ; i++) {
13270 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
13271 /* directory entry dependency */
13280 bmsafemap = WK_BMSAFEMAP(wk);
13281 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) {
13282 /* Add reference dependency. */
13287 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) {
13288 /* Allocate block dependency. */
13296 freeblks = WK_FREEBLKS(wk);
13297 if (LIST_FIRST(&freeblks->fb_jblkdephd)) {
13298 /* Freeblk journal dependency. */
13305 case D_ALLOCDIRECT:
13307 newblk = WK_NEWBLK(wk);
13308 if (newblk->nb_jnewblk) {
13309 /* Journal allocate dependency. */
13317 mkdir = WK_MKDIR(wk);
13318 if (mkdir->md_jaddref) {
13319 /* Journal reference dependency. */
13331 /* never a dependency on these blocks */
13335 panic("softdep_count_dependencies: Unexpected type %s",
13336 TYPENAME(wk->wk_type));
13346 * Acquire exclusive access to a buffer.
13347 * Must be called with a locked mtx parameter.
13348 * Return acquired buffer or NULL on failure.
13350 static struct buf *
13351 getdirtybuf(bp, mtx, waitfor)
13358 mtx_assert(mtx, MA_OWNED);
13359 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) {
13360 if (waitfor != MNT_WAIT)
13362 error = BUF_LOCK(bp,
13363 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, mtx);
13365 * Even if we sucessfully acquire bp here, we have dropped
13366 * mtx, which may violates our guarantee.
13370 else if (error != ENOLCK)
13371 panic("getdirtybuf: inconsistent lock: %d", error);
13375 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
13376 if (mtx == &lk && waitfor == MNT_WAIT) {
13378 BO_LOCK(bp->b_bufobj);
13380 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
13381 bp->b_vflags |= BV_BKGRDWAIT;
13382 msleep(&bp->b_xflags, BO_MTX(bp->b_bufobj),
13383 PRIBIO | PDROP, "getbuf", 0);
13385 BO_UNLOCK(bp->b_bufobj);
13390 if (waitfor != MNT_WAIT)
13393 * The mtx argument must be bp->b_vp's mutex in
13396 #ifdef DEBUG_VFS_LOCKS
13397 if (bp->b_vp->v_type != VCHR)
13398 ASSERT_BO_LOCKED(bp->b_bufobj);
13400 bp->b_vflags |= BV_BKGRDWAIT;
13401 msleep(&bp->b_xflags, mtx, PRIBIO, "getbuf", 0);
13404 if ((bp->b_flags & B_DELWRI) == 0) {
13414 * Check if it is safe to suspend the file system now. On entry,
13415 * the vnode interlock for devvp should be held. Return 0 with
13416 * the mount interlock held if the file system can be suspended now,
13417 * otherwise return EAGAIN with the mount interlock held.
13420 softdep_check_suspend(struct mount *mp,
13421 struct vnode *devvp,
13423 int softdep_accdeps,
13424 int secondary_writes,
13425 int secondary_accwrites)
13428 struct ufsmount *ump;
13431 ump = VFSTOUFS(mp);
13432 bo = &devvp->v_bufobj;
13433 ASSERT_BO_LOCKED(bo);
13436 if (!TRY_ACQUIRE_LOCK(&lk)) {
13444 if (mp->mnt_secondary_writes != 0) {
13447 msleep(&mp->mnt_secondary_writes,
13449 (PUSER - 1) | PDROP, "secwr", 0);
13457 * Reasons for needing more work before suspend:
13458 * - Dirty buffers on devvp.
13459 * - Softdep activity occurred after start of vnode sync loop
13460 * - Secondary writes occurred after start of vnode sync loop
13463 if (bo->bo_numoutput > 0 ||
13464 bo->bo_dirty.bv_cnt > 0 ||
13465 softdep_deps != 0 ||
13466 ump->softdep_deps != 0 ||
13467 softdep_accdeps != ump->softdep_accdeps ||
13468 secondary_writes != 0 ||
13469 mp->mnt_secondary_writes != 0 ||
13470 secondary_accwrites != mp->mnt_secondary_accwrites)
13479 * Get the number of dependency structures for the file system, both
13480 * the current number and the total number allocated. These will
13481 * later be used to detect that softdep processing has occurred.
13484 softdep_get_depcounts(struct mount *mp,
13485 int *softdep_depsp,
13486 int *softdep_accdepsp)
13488 struct ufsmount *ump;
13490 ump = VFSTOUFS(mp);
13492 *softdep_depsp = ump->softdep_deps;
13493 *softdep_accdepsp = ump->softdep_accdeps;
13498 * Wait for pending output on a vnode to complete.
13499 * Must be called with vnode lock and interlock locked.
13501 * XXX: Should just be a call to bufobj_wwait().
13509 bo = &vp->v_bufobj;
13510 ASSERT_VOP_LOCKED(vp, "drain_output");
13511 ASSERT_BO_LOCKED(bo);
13513 while (bo->bo_numoutput) {
13514 bo->bo_flag |= BO_WWAIT;
13515 msleep((caddr_t)&bo->bo_numoutput,
13516 BO_MTX(bo), PRIBIO + 1, "drainvp", 0);
13521 * Called whenever a buffer that is being invalidated or reallocated
13522 * contains dependencies. This should only happen if an I/O error has
13523 * occurred. The routine is called with the buffer locked.
13526 softdep_deallocate_dependencies(bp)
13530 if ((bp->b_ioflags & BIO_ERROR) == 0)
13531 panic("softdep_deallocate_dependencies: dangling deps");
13532 softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error);
13533 panic("softdep_deallocate_dependencies: unrecovered I/O error");
13537 * Function to handle asynchronous write errors in the filesystem.
13540 softdep_error(func, error)
13545 /* XXX should do something better! */
13546 printf("%s: got error %d while accessing filesystem\n", func, error);
13552 inodedep_print(struct inodedep *inodedep, int verbose)
13554 db_printf("%p fs %p st %x ino %jd inoblk %jd delta %d nlink %d"
13556 inodedep, inodedep->id_fs, inodedep->id_state,
13557 (intmax_t)inodedep->id_ino,
13558 (intmax_t)fsbtodb(inodedep->id_fs,
13559 ino_to_fsba(inodedep->id_fs, inodedep->id_ino)),
13560 inodedep->id_nlinkdelta, inodedep->id_savednlink,
13561 inodedep->id_savedino1);
13566 db_printf("\tpendinghd %p, bufwait %p, inowait %p, inoreflst %p, "
13568 LIST_FIRST(&inodedep->id_pendinghd),
13569 LIST_FIRST(&inodedep->id_bufwait),
13570 LIST_FIRST(&inodedep->id_inowait),
13571 TAILQ_FIRST(&inodedep->id_inoreflst),
13572 inodedep->id_mkdiradd);
13573 db_printf("\tinoupdt %p, newinoupdt %p, extupdt %p, newextupdt %p\n",
13574 TAILQ_FIRST(&inodedep->id_inoupdt),
13575 TAILQ_FIRST(&inodedep->id_newinoupdt),
13576 TAILQ_FIRST(&inodedep->id_extupdt),
13577 TAILQ_FIRST(&inodedep->id_newextupdt));
13580 DB_SHOW_COMMAND(inodedep, db_show_inodedep)
13583 if (have_addr == 0) {
13584 db_printf("Address required\n");
13587 inodedep_print((struct inodedep*)addr, 1);
13590 DB_SHOW_COMMAND(inodedeps, db_show_inodedeps)
13592 struct inodedep_hashhead *inodedephd;
13593 struct inodedep *inodedep;
13597 fs = have_addr ? (struct fs *)addr : NULL;
13598 for (cnt = 0; cnt < inodedep_hash; cnt++) {
13599 inodedephd = &inodedep_hashtbl[cnt];
13600 LIST_FOREACH(inodedep, inodedephd, id_hash) {
13601 if (fs != NULL && fs != inodedep->id_fs)
13603 inodedep_print(inodedep, 0);
13608 DB_SHOW_COMMAND(worklist, db_show_worklist)
13610 struct worklist *wk;
13612 if (have_addr == 0) {
13613 db_printf("Address required\n");
13616 wk = (struct worklist *)addr;
13617 printf("worklist: %p type %s state 0x%X\n",
13618 wk, TYPENAME(wk->wk_type), wk->wk_state);
13621 DB_SHOW_COMMAND(workhead, db_show_workhead)
13623 struct workhead *wkhd;
13624 struct worklist *wk;
13627 if (have_addr == 0) {
13628 db_printf("Address required\n");
13631 wkhd = (struct workhead *)addr;
13632 wk = LIST_FIRST(wkhd);
13633 for (i = 0; i < 100 && wk != NULL; i++, wk = LIST_NEXT(wk, wk_list))
13634 db_printf("worklist: %p type %s state 0x%X",
13635 wk, TYPENAME(wk->wk_type), wk->wk_state);
13637 db_printf("workhead overflow");
13642 DB_SHOW_COMMAND(mkdirs, db_show_mkdirs)
13644 struct jaddref *jaddref;
13645 struct diradd *diradd;
13646 struct mkdir *mkdir;
13648 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) {
13649 diradd = mkdir->md_diradd;
13650 db_printf("mkdir: %p state 0x%X dap %p state 0x%X",
13651 mkdir, mkdir->md_state, diradd, diradd->da_state);
13652 if ((jaddref = mkdir->md_jaddref) != NULL)
13653 db_printf(" jaddref %p jaddref state 0x%X",
13654 jaddref, jaddref->ja_state);
13661 #endif /* SOFTUPDATES */