2 * Copyright (c) 2010-2012 Michihiro NAKAJIMA
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 __FBSDID("$FreeBSD$");
28 #ifdef HAVE_SYS_IOCTL_H
29 #include <sys/ioctl.h>
31 #ifdef HAVE_SYS_PARAM_H
32 #include <sys/param.h>
43 #ifdef HAVE_LINUX_TYPES_H
44 #include <linux/types.h>
46 #ifdef HAVE_LINUX_FIEMAP_H
47 #include <linux/fiemap.h>
49 #ifdef HAVE_LINUX_FS_H
53 /* The logic to compare sparse file data read from disk with the
54 * specification is a little involved. Set to 1 to have the progress
59 * NOTE: On FreeBSD and Solaris, this test needs ZFS.
60 * You may perform this test as
61 * 'TMPDIR=<a directory on the ZFS> libarchive_test'.
65 enum { DATA, HOLE, END } type;
69 static void create_sparse_file(const char *, const struct sparse *);
71 #if defined(__APPLE__)
72 /* On APFS holes need to be at least 4096x4097 bytes */
73 #define MIN_HOLE 16781312
75 /* Elsewhere we work with 4096*10 bytes */
76 #define MIN_HOLE 409600
79 #if defined(_WIN32) && !defined(__CYGWIN__)
82 * Create a sparse file on Windows.
85 #if !defined(PATH_MAX)
86 #define PATH_MAX MAX_PATH
88 #if !defined(__BORLANDC__)
89 #define getcwd _getcwd
93 is_sparse_supported(const char *path)
95 char root[MAX_PATH+1];
101 strncpy(root, path, sizeof(root)-1);
102 if (((root[0] >= 'c' && root[0] <= 'z') ||
103 (root[0] >= 'C' && root[0] <= 'Z')) &&
105 (root[2] == '\\' || root[2] == '/'))
109 assertEqualInt((r = GetVolumeInformation(root, vol,
110 sizeof(vol), NULL, NULL, &flags, sys, sizeof(sys))), 1);
111 return (r != 0 && (flags & FILE_SUPPORTS_SPARSE_FILES) != 0);
115 create_sparse_file(const char *path, const struct sparse *s)
121 memset(buff, ' ', sizeof(buff));
123 handle = CreateFileA(path, GENERIC_WRITE, 0,
124 NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL,
126 assert(handle != INVALID_HANDLE_VALUE);
127 assert(DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0,
128 NULL, 0, &dmy, NULL) != 0);
130 uint64_t offsetSoFar = 0;
132 while (s->type != END) {
133 if (s->type == HOLE) {
134 LARGE_INTEGER fileOffset, beyondOffset, distanceToMove;
135 fileOffset.QuadPart = offsetSoFar;
136 beyondOffset.QuadPart = offsetSoFar + s->size;
137 distanceToMove.QuadPart = s->size;
139 FILE_ZERO_DATA_INFORMATION zeroInformation;
140 zeroInformation.FileOffset = fileOffset;
141 zeroInformation.BeyondFinalZero = beyondOffset;
144 assert(SetFilePointerEx(handle, distanceToMove,
145 NULL, FILE_CURRENT) != 0);
146 assert(SetEndOfFile(handle) != 0);
147 assert(DeviceIoControl(handle, FSCTL_SET_ZERO_DATA, &zeroInformation,
148 sizeof(FILE_ZERO_DATA_INFORMATION), NULL, 0, &bytesReturned, NULL) != 0);
155 if (size > sizeof(buff))
159 assert(WriteFile(handle, buff, w, &wr, NULL) != 0);
163 offsetSoFar += s->size;
166 assertEqualInt(CloseHandle(handle), 1);
171 #if defined(HAVE_LINUX_FIEMAP_H)
173 * FIEMAP, which can detect 'hole' of a sparse file, has
174 * been supported from 2.6.28
178 is_sparse_supported_fiemap(const char *path)
180 const struct sparse sparse_file[] = {
181 /* This hole size is too small to create a sparse
182 * files for almost filesystem. */
183 { HOLE, 1024 }, { DATA, 10240 },
189 const char *testfile = "can_sparse";
191 (void)path; /* UNUSED */
192 memset(buff, 0, sizeof(buff));
193 create_sparse_file(testfile, sparse_file);
194 fd = open(testfile, O_RDWR);
197 fm = (struct fiemap *)buff;
199 fm->fm_length = ~0ULL;;
200 fm->fm_flags = FIEMAP_FLAG_SYNC;
201 fm->fm_extent_count = (sizeof(buff) - sizeof(*fm))/
202 sizeof(struct fiemap_extent);
203 r = ioctl(fd, FS_IOC_FIEMAP, fm);
209 #if !defined(SEEK_HOLE) || !defined(SEEK_DATA)
211 is_sparse_supported(const char *path)
213 return is_sparse_supported_fiemap(path);
218 #if defined(_PC_MIN_HOLE_SIZE)
221 * FreeBSD and Solaris can detect 'hole' of a sparse file
222 * through lseek(HOLE) on ZFS. (UFS does not support yet)
226 is_sparse_supported(const char *path)
228 return (pathconf(path, _PC_MIN_HOLE_SIZE) > 0);
231 #elif defined(SEEK_HOLE) && defined(SEEK_DATA)
234 is_sparse_supported(const char *path)
236 const struct sparse sparse_file[] = {
237 /* This hole size is too small to create a sparse
238 * files for almost filesystem. */
239 { HOLE, 1024 }, { DATA, 10240 },
243 const char *testfile = "can_sparse";
245 (void)path; /* UNUSED */
246 create_sparse_file(testfile, sparse_file);
247 fd = open(testfile, O_RDWR);
250 r = lseek(fd, 0, SEEK_HOLE);
253 #if defined(HAVE_LINUX_FIEMAP_H)
255 return (is_sparse_supported_fiemap(path));
260 #elif !defined(HAVE_LINUX_FIEMAP_H)
263 * Other system may do not have the API such as lseek(HOLE),
264 * which detect 'hole' of a sparse file.
268 is_sparse_supported(const char *path)
270 (void)path; /* UNUSED */
277 * Create a sparse file on POSIX like system.
281 create_sparse_file(const char *path, const struct sparse *s)
285 uint64_t total_size = 0;
286 const struct sparse *cur = s;
288 memset(buff, ' ', sizeof(buff));
289 assert((fd = open(path, O_CREAT | O_WRONLY, 0600)) != -1);
291 /* Handle holes at the end by extending the file */
292 while (cur->type != END) {
293 total_size += cur->size;
296 assert(ftruncate(fd, total_size) != -1);
298 while (s->type != END) {
299 if (s->type == HOLE) {
300 assert(lseek(fd, s->size, SEEK_CUR) != (off_t)-1);
306 if (size > sizeof(buff))
310 assert(write(fd, buff, w) != (ssize_t)-1);
322 * Sparse test with directory traversals.
325 verify_sparse_file(struct archive *a, const char *path,
326 const struct sparse *sparse, int expected_holes)
328 struct archive_entry *ae;
331 int64_t offset, expected_offset, last_offset;
334 create_sparse_file(path, sparse);
335 assert((ae = archive_entry_new()) != NULL);
336 assertEqualIntA(a, ARCHIVE_OK, archive_read_disk_open(a, path));
337 assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header2(a, ae));
341 while (ARCHIVE_OK == archive_read_data_block(a, &buff, &bytes_read,
343 const char *start = buff;
345 fprintf(stderr, "%s: bytes_read=%d offset=%d\n", path, (int)bytes_read, (int)offset);
347 if (offset > last_offset) {
350 /* Blocks entirely before the data we just read. */
351 while (expected_offset + (int64_t)sparse->size < offset) {
353 fprintf(stderr, " skipping expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size);
356 assert(sparse->type == HOLE);
357 expected_offset += sparse->size;
360 /* Block that overlaps beginning of data */
361 if (expected_offset < offset
362 && expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) {
363 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size;
365 fprintf(stderr, " overlapping hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size);
367 /* Must be a hole, overlap must be filled with '\0' */
368 if (assert(sparse->type == HOLE)) {
369 assertMemoryFilledWith(start, end - start, '\0');
372 expected_offset += sparse->size;
375 /* Blocks completely contained in data we just read. */
376 while (expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) {
377 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size;
378 if (sparse->type == HOLE) {
380 fprintf(stderr, " contained hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size);
383 /* verify data corresponding to hole is '\0' */
384 if (end > (const char *)buff + bytes_read) {
385 end = (const char *)buff + bytes_read;
387 assertMemoryFilledWith(start, end - start, '\0');
389 expected_offset += sparse->size;
391 } else if (sparse->type == DATA) {
393 fprintf(stderr, " contained data expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size);
395 /* verify data corresponding to hole is ' ' */
396 if (assert(expected_offset + sparse->size <= offset + bytes_read)) {
397 assert(start == (const char *)buff + (size_t)(expected_offset - offset));
398 assertMemoryFilledWith(start, end - start, ' ');
401 expected_offset += sparse->size;
407 /* Block that overlaps end of data */
408 if (expected_offset < offset + (int64_t)bytes_read) {
409 const char *end = (const char *)buff + bytes_read;
411 fprintf(stderr, " trailing overlap expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size);
413 /* Must be a hole, overlap must be filled with '\0' */
414 if (assert(sparse->type == HOLE)) {
415 assertMemoryFilledWith(start, end - start, '\0');
418 last_offset = offset + bytes_read;
420 /* Count a hole at EOF? */
421 if (last_offset < archive_entry_size(ae)) {
425 /* Verify blocks after last read */
426 while (sparse->type == HOLE) {
427 expected_offset += sparse->size;
430 assert(sparse->type == END);
431 assertEqualInt(expected_offset, archive_entry_size(ae));
434 assertEqualInt(holes_seen, expected_holes);
436 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
437 archive_entry_free(ae);
440 #if defined(_WIN32) && !defined(__CYGWIN__)
446 * Sparse test without directory traversals.
449 verify_sparse_file2(struct archive *a, const char *path,
450 const struct sparse *sparse, int blocks, int preopen)
452 struct archive_entry *ae;
455 (void)sparse; /* UNUSED */
456 assert((ae = archive_entry_new()) != NULL);
457 archive_entry_set_pathname(ae, path);
459 fd = open(path, O_RDONLY | O_BINARY);
462 assertEqualIntA(a, ARCHIVE_OK,
463 archive_read_disk_entry_from_file(a, ae, fd, NULL));
466 /* Verify the number of holes only, not its offset nor its
467 * length because those alignments are deeply dependence on
470 assertEqualInt(blocks, archive_entry_sparse_count(ae));
471 archive_entry_free(ae);
475 test_sparse_whole_file_data(void)
477 struct archive_entry *ae;
481 assert((ae = archive_entry_new()) != NULL);
482 archive_entry_set_size(ae, 1024*10);
485 * Add sparse block data up to the file size.
488 for (i = 0; i < 10; i++) {
489 archive_entry_sparse_add_entry(ae, offset, 1024);
493 failure("There should be no sparse");
494 assertEqualInt(0, archive_entry_sparse_count(ae));
495 archive_entry_free(ae);
498 DEFINE_TEST(test_sparse_basic)
502 const char *skip_sparse_tests;
504 * The alignment of the hole of sparse files deeply depends
505 * on filesystem. In my experience, sparse_file2 test with
506 * 204800 bytes hole size did not pass on ZFS and the result
507 * of that test seemed the size was too small, thus you should
508 * keep a hole size more than 409600 bytes to pass this test
511 const struct sparse sparse_file0[] = {
513 { DATA, 1024 }, { HOLE, MIN_HOLE + 1638400 },
514 // 2049024 // 2051072
515 { DATA, 2048 }, { HOLE, MIN_HOLE + 1638400 },
516 // 4099072 // 4103168
517 { DATA, 4096 }, { HOLE, MIN_HOLE + 20070400 },
518 // 24583168 // 24591360
519 { DATA, 8192 }, { HOLE, MIN_HOLE + 204390400 },
520 // 229391360 // 229391361
521 { DATA, 1 }, { END, 0 }
523 const struct sparse sparse_file1[] = {
524 { HOLE, MIN_HOLE }, { DATA, 1 },
525 { HOLE, MIN_HOLE }, { DATA, 1 },
526 { HOLE, MIN_HOLE }, { END, 0 }
528 const struct sparse sparse_file2[] = {
529 { HOLE, MIN_HOLE }, { DATA, 1024 },
530 { HOLE, MIN_HOLE + 409600 * 1 }, { DATA, 1024 },
531 { HOLE, MIN_HOLE + 409600 * 2 }, { DATA, 1024 },
532 { HOLE, MIN_HOLE + 409600 * 3 }, { DATA, 1024 },
533 { HOLE, MIN_HOLE + 409600 * 4 }, { DATA, 1024 },
534 { HOLE, MIN_HOLE + 409600 * 5 }, { DATA, 1024 },
535 { HOLE, MIN_HOLE + 409600 * 6 }, { DATA, 1024 },
536 { HOLE, MIN_HOLE + 409600 * 7 }, { DATA, 1024 },
537 { HOLE, MIN_HOLE + 409600 * 8 }, { DATA, 1024 },
538 { HOLE, MIN_HOLE + 409600 * 9}, { DATA, 1024 },/* 10 */
539 { HOLE, MIN_HOLE }, { DATA, 1024 * 1 },
540 { HOLE, MIN_HOLE + 409600 * 1 }, { DATA, 1024 * 2 },
541 { HOLE, MIN_HOLE + 409600 * 2 }, { DATA, 1024 * 3 },
542 { HOLE, MIN_HOLE + 409600 * 3 }, { DATA, 1024 * 4 },
543 { HOLE, MIN_HOLE + 409600 * 4 }, { DATA, 1024 * 5 },
544 { HOLE, MIN_HOLE + 409600 * 5 }, { DATA, 1024 * 6 },
545 { HOLE, MIN_HOLE + 409600 * 6 }, { DATA, 1024 * 7 },
546 { HOLE, MIN_HOLE + 409600 * 7 }, { DATA, 1024 * 8 },
547 { HOLE, MIN_HOLE + 409600 * 8 }, { DATA, 1024 * 9 },
548 { HOLE, MIN_HOLE + 409600 * 9}, { DATA, 1024 * 10},/* 20 */
551 const struct sparse sparse_file3[] = {
552 /* This hole size is too small to create a sparse file */
553 { HOLE, 1 }, { DATA, 10240 },
554 { HOLE, 1 }, { DATA, 10240 },
555 { HOLE, 1 }, { DATA, 10240 },
558 const struct sparse sparse_file4[] = {
559 { DATA, 4096 }, { HOLE, 0xc0000000 },
560 /* This hole overflows the offset if stored in 32 bits. */
561 { DATA, 4096 }, { HOLE, 0x50000000 },
566 * Test for the case that sparse data indicates just the whole file
569 test_sparse_whole_file_data();
571 skip_sparse_tests = getenv("SKIP_TEST_SPARSE");
572 if (skip_sparse_tests != NULL) {
573 skipping("Skipping sparse tests due to SKIP_TEST_SPARSE "
574 "environment variable");
578 /* Check if the filesystem where CWD on can
579 * report the number of the holes of a sparse file. */
581 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */
583 cwd = getcwd(NULL, 0);
585 if (!assert(cwd != NULL))
587 if (!is_sparse_supported(cwd)) {
589 skipping("This filesystem or platform do not support "
590 "the reporting of the holes of a sparse file through "
591 "API such as lseek(HOLE)");
596 * Get sparse data through directory traversals.
598 assert((a = archive_read_disk_new()) != NULL);
600 verify_sparse_file(a, "file0", sparse_file0, 4);
601 verify_sparse_file(a, "file1", sparse_file1, 3);
602 verify_sparse_file(a, "file2", sparse_file2, 20);
603 /* Encoded non sparse; expect a data block but no sparse entries. */
604 verify_sparse_file(a, "file3", sparse_file3, 0);
605 verify_sparse_file(a, "file4", sparse_file4, 2);
607 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
610 * Get sparse data through archive_read_disk_entry_from_file().
612 assert((a = archive_read_disk_new()) != NULL);
614 verify_sparse_file2(a, "file0", sparse_file0, 5, 0);
615 verify_sparse_file2(a, "file0", sparse_file0, 5, 1);
617 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
621 DEFINE_TEST(test_fully_sparse_files)
625 const char *skip_sparse_tests;
627 const struct sparse sparse_file[] = {
628 { HOLE, MIN_HOLE }, { END, 0 }
631 skip_sparse_tests = getenv("SKIP_TEST_SPARSE");
632 if (skip_sparse_tests != NULL) {
633 skipping("Skipping sparse tests due to SKIP_TEST_SPARSE "
634 "environment variable");
638 /* Check if the filesystem where CWD on can
639 * report the number of the holes of a sparse file. */
641 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */
643 cwd = getcwd(NULL, 0);
645 if (!assert(cwd != NULL))
647 if (!is_sparse_supported(cwd)) {
649 skipping("This filesystem or platform do not support "
650 "the reporting of the holes of a sparse file through "
651 "API such as lseek(HOLE)");
655 assert((a = archive_read_disk_new()) != NULL);
657 /* Fully sparse files are encoded with a zero-length "data" block. */
658 verify_sparse_file(a, "file0", sparse_file, 1);
660 assertEqualInt(ARCHIVE_OK, archive_read_free(a));