/* Copyright 2012, Unpublished Work of Technologic Systems * All Rights Reserved. * * THIS WORK IS AN UNPUBLISHED WORK AND CONTAINS CONFIDENTIAL, * PROPRIETARY AND TRADE SECRET INFORMATION OF TECHNOLOGIC SYSTEMS. * ACCESS TO THIS WORK IS RESTRICTED TO (I) TECHNOLOGIC SYSTEMS EMPLOYEES * WHO HAVE A NEED TO KNOW TO PERFORM TASKS WITHIN THE SCOPE OF THEIR * ASSIGNMENTS AND (II) ENTITIES OTHER THAN TECHNOLOGIC SYSTEMS WHO * HAVE ENTERED INTO APPROPRIATE LICENSE AGREEMENTS. NO PART OF THIS * WORK MAY BE USED, PRACTICED, PERFORMED, COPIED, DISTRIBUTED, REVISED, * MODIFIED, TRANSLATED, ABRIDGED, CONDENSED, EXPANDED, COLLECTED, * COMPILED,LINKED,RECAST, TRANSFORMED, ADAPTED IN ANY FORM OR BY ANY * MEANS,MANUAL, MECHANICAL, CHEMICAL, ELECTRICAL, ELECTRONIC, OPTICAL, * BIOLOGICAL, OR OTHERWISE WITHOUT THE PRIOR WRITTEN PERMISSION AND * CONSENT OF TECHNOLOGIC SYSTEMS . ANY USE OR EXPLOITATION OF THIS WORK * WITHOUT THE PRIOR WRITTEN CONSENT OF TECHNOLOGIC SYSTEMS COULD * SUBJECT THE PERPETRATOR TO CRIMINAL AND CIVIL LIABILITY. */ /* To compile dblstorctl, use the appropriate cross compiler and run the * command: * * gcc -Wall -O -mcpu=arm9 -o dblstorctl dblstorctl.c * * dblstor.c should be in the same directory as dblstorctl.c * * On uclibc based initrd's, the following additional gcc options are * necessary: -Wl,--rpath,/slib -Wl,-dynamic-linker,/slib/ld-uClibc.so.0 */ const char copyright[] = "Copyright (c) Technologic Systems - " __DATE__ ; #ifdef __linux__ #define _LARGEFILE64_SOURCE #define _FILE_OFFSET_BITS 64 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __linux__ #include #endif #ifdef __APPLE__ #include #endif #include "dblstor.c" #ifndef TEMP_FAILURE_RETRY # define TEMP_FAILURE_RETRY(expression) \ (__extension__ \ ({ long int __result; \ do __result = (long int) (expression); \ while (__result == -1L && errno == EINTR); \ __result; })) #endif static struct dblstor cb; static unsigned int ll_size, ticks; static int ll_fd, fb_fd; #ifdef __linux__ static off64_t ll_fd_offset, fb_fd_offset; #else static off_t ll_fd_offset, fb_fd_offset; #endif static int opt_foreground; static int opt_savebadblocks = 0, no_update_curtime = 0; static int nbdbufsz; static unsigned char *nbdbuf; static int ll_read(void *, unsigned int, unsigned char *, int); static int ll_write(void *, unsigned int, unsigned char *, int); static int dly_ll_write(void *, unsigned int, unsigned char *, int); static int ll_nbdread(unsigned int, unsigned char *, int); static int ll_nbdwrite(unsigned int, unsigned char *, int); static int nbdidle(void); static void nbdserver(char *, int, struct in_addr) __attribute__((noreturn)); static void alarmsig(int); struct nbd_request { unsigned int magic; char type[4]; char handle[8]; unsigned int fromhi; unsigned int fromlo; unsigned int len; }; struct nbd_reply { char magic[4]; unsigned int error; char handle[8]; }; struct dmesg { /* This struct is stored in SHM */ volatile int lock; int put; int get; char buf[16384]; #define NBDPID 1 #define NBD_READREQS 2 #define NBD_READ_BLKS 3 #define NBD_WRITEREQS 4 #define NBD_WRITE_BLKS 5 #define NBD_SEEK_PAST_EOF_ERRS 6 #define DBLSTOR_TAINTSTATUS 7 #define DBLSTOR_CURSEQ 8 #define DBLSTOR_RESILVER_PCT_DONE 9 #define DBLSTOR_ERRORS 10 #define DBLSTOR_UNRECOVERABLE_ERRORS 11 #define DBLSTOR_CONFLICTS 12 #define DBLSTOR_SIZE 13 #define DBLSTOR_FB_OFFSET 14 unsigned int int32[32]; unsigned long long int64[32]; }; static struct dmesg *shm_dmesg; static void printk(struct dmesg *, const char *, ...); static unsigned char cavium_mbr[512] = { 0x28,0x31,0x9f,0xe5,0x08,0xd0,0x4d,0xe2,0x01,0x60,0xa0,0xe1,0x00,0x80,0xa0,0xe1, 0x03,0x00,0x93,0xe8,0x0d,0x30,0xa0,0xe1,0x03,0x00,0x83,0xe8,0x10,0x01,0x9f,0xe5, 0x02,0x70,0xa0,0xe1,0x00,0xa0,0x9d,0xe5,0x08,0x41,0x9f,0xe5,0x0f,0xe0,0xa0,0xe1, 0x06,0xf0,0xa0,0xe1,0x00,0x50,0xa0,0xe3,0x12,0x00,0x00,0xea,0x04,0x30,0xd4,0xe5, 0xda,0x00,0x53,0xe3,0x0e,0x00,0x00,0x1a,0xb8,0x10,0xd4,0xe1,0xbc,0xc0,0xd4,0xe1, 0xba,0x00,0xd4,0xe1,0xbe,0x20,0xd4,0xe1,0x08,0xe0,0x8d,0xe2,0x05,0x31,0x8e,0xe0, 0x00,0x08,0x81,0xe1,0x02,0x28,0x8c,0xe1,0x08,0x10,0x13,0xe5,0x0f,0xe0,0xa0,0xe1, 0x08,0xf0,0xa0,0xe1,0xb8,0x00,0x9f,0xe5,0x01,0x50,0x85,0xe2,0x0f,0xe0,0xa0,0xe1, 0x06,0xf0,0xa0,0xe1,0x10,0x40,0x84,0xe2,0xac,0x30,0x9f,0xe5,0x03,0x00,0x54,0xe1, 0xe9,0xff,0xff,0x1a,0x0f,0xe0,0xa0,0xe1,0x07,0xf0,0xa0,0xe1,0x01,0x00,0x55,0xe3, 0x98,0x30,0x9f,0xc5,0x78,0x20,0xe0,0xc3,0x06,0x20,0xc3,0xc5,0x90,0xc0,0x9f,0xe5, 0x01,0x1c,0xa0,0xe3,0x02,0x00,0x5c,0xe5,0x88,0xe0,0x9f,0xe5,0x01,0x20,0x5c,0xe5, 0x3f,0x30,0x00,0xe2,0x03,0x20,0xce,0xe7,0x80,0x00,0x10,0xe3,0x00,0x30,0xde,0x15, 0x00,0x20,0xa0,0x13,0x03,0xe1,0xa0,0x11,0x04,0x00,0x00,0x1a,0x06,0x00,0x00,0xea, 0x60,0x30,0x9f,0xe5,0x02,0x30,0xd3,0xe7,0x02,0x30,0xc1,0xe7,0x01,0x20,0x82,0xe2, 0x0e,0x00,0x52,0xe1,0xf9,0xff,0xff,0x3a,0x0e,0x10,0x81,0xe0,0x40,0x00,0x10,0xe3, 0x02,0xc0,0x8c,0xe2,0xea,0xff,0xff,0x0a,0x3c,0x30,0x9f,0xe5,0x00,0x00,0xa0,0xe3, 0x00,0x10,0x83,0xe5,0x00,0x00,0x81,0xe5,0x01,0x2c,0xa0,0xe3,0x2c,0x10,0x9f,0xe5, 0x0f,0xe0,0xa0,0xe1,0x0a,0xf0,0xa0,0xe1,0x08,0xd0,0x8d,0xe2,0x0e,0xf0,0xa0,0xe1, 0x54,0x41,0x00,0x00,0x5c,0x41,0x00,0x00,0xbe,0x41,0x00,0x00,0xfe,0x41,0x00,0x00, 0x60,0x41,0x00,0x00,0x62,0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x14,0x42,0x00,0x00, 0xd1,0x07,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x00,0x00,0x01,0x2e,0x0d,0x0a,0x00, 0x00,0x05,0x04,0x01,0x06,0x41,0xc7,0x54,0x04,0x05,0x06,0x42,0x0b,0x01,0x0e,0x40, 0xcf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0xda,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xff,0x0f,0x00,0x00,0x00,0x00, 0x00,0x00,0xda,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00, 0x01,0x00,0x83,0x00,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0xe0,0x07,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x55,0xaa, }; static inline void nbdwrite(int fd, unsigned int sector, int len) { int ret; unsigned char *buf = nbdbuf; int set_nonblock_on_exit = 0; while (len > nbdbufsz) { nbdwrite(fd, sector, nbdbufsz); sector += nbdbufsz; len -= nbdbufsz; } ret = ll_nbdread(sector, buf, len); // assert (ret == 0); if (ret != 0) bzero(buf, len * 512); len = len << 9; while (len) { ret = TEMP_FAILURE_RETRY(write(fd, buf, len)); if (ret == -1 && errno == EAGAIN) { fcntl(fd, F_SETFL, 0); set_nonblock_on_exit = 1; continue; } assert (ret > 0); len -= ret; buf += ret; } if (set_nonblock_on_exit) fcntl(fd, F_SETFL, O_NONBLOCK); } static inline void nbdread(int fd, unsigned int sector, int len) { int ret; unsigned char *buf = nbdbuf; int rem; int set_nonblock_on_exit = 0; while (len > nbdbufsz) { nbdread(fd, sector, nbdbufsz); sector += nbdbufsz; len -= nbdbufsz; } rem = len << 9; while (rem) { ret = TEMP_FAILURE_RETRY(read(fd, buf, rem)); if (ret == -1 && errno == EAGAIN) { fcntl(fd, F_SETFL, 0); set_nonblock_on_exit = 1; continue; } assert (ret > 0); rem -= ret; buf += ret; } ret = ll_nbdwrite(sector, nbdbuf, len); assert (ret == 0); if (set_nonblock_on_exit) fcntl(fd, F_SETFL, O_NONBLOCK); } static inline int xread(int fd, void *d, int len) { int olen = len; unsigned char *buf = d; int set_nonblock_on_exit = 0; if (buf == NULL) { buf = nbdbuf; while (len > (nbdbufsz << 9)) { xread(fd, buf, nbdbufsz); len -= nbdbufsz; } if (len) xread(fd, buf, len); } while (len > 0) { int x = TEMP_FAILURE_RETRY(read(fd, buf, len)); if (x == -1 && errno == EAGAIN) { fcntl(fd, F_SETFL, 0); set_nonblock_on_exit = 1; continue; } if (x == 0 || x < 0) return 0; buf += x; len -= x; } if (set_nonblock_on_exit) fcntl(fd, F_SETFL, O_NONBLOCK); return olen; } static inline int xwrite(int fd, void *d, int len) { int olen = len; unsigned char *buf = d; int set_nonblock_on_exit = 0; while (len > 0) { int x = TEMP_FAILURE_RETRY(write(fd, buf, len)); if (x == -1 && errno == EAGAIN) { fcntl(fd, F_SETFL, 0); set_nonblock_on_exit = 1; continue; } if (x == 0 || x < 0) return 0; buf += x; len -= x; } if (set_nonblock_on_exit) fcntl(fd, F_SETFL, O_NONBLOCK); return olen; } static int get_ptbl_offs(int part, int *type, int *sz) { unsigned char mbr[512]; int ret, ext = 0; ret = dblstor_read(&cb, 0, mbr, 1); if (ret != 0) return -1; if (part > 4) { /* extended partition */ int i, oext; for (i = 0; i < 4; i++) if (mbr[0x1be + (16 * i) + 4] == 5) break; if (i == 4) return -1; oext = mbr[0x1be + (16 * i) + 8]; oext |= mbr[0x1be + (16 * i) + 9] << 8; oext |= mbr[0x1be + (16 * i) + 10] << 16; oext |= mbr[0x1be + (16 * i) + 11] << 24; ext = oext; ret = dblstor_read(&cb, oext, mbr, 1); if (ret != 0) return -1; /* We now have the first EBR in 'mbr' from sector 'oext' */ /* Skip over 'part' - 5 EBRs -- we don't want them */ while (part > 5) { int n; n = mbr[0x1d6]; n |= mbr[0x1d7] << 8; n |= mbr[0x1d8] << 16; n |= mbr[0x1d9] << 24; ext = oext + n; if (mbr[0x1d2] == 0) return -1; ret = dblstor_read(&cb, ext, mbr, 1); if (ret != 0) return -1; part--; } part -= 4; } part--; ret = mbr[0x1be + (16 * part) + 8]; ret |= mbr[0x1be + (16 * part) + 9] << 8; ret |= mbr[0x1be + (16 * part) + 10] << 16; ret |= mbr[0x1be + (16 * part) + 11] << 24; ret += ext; if (type) *type = mbr[0x1be + (16 * part) + 4]; if (sz) { *sz = mbr[0x1be + (16 * part) + 12]; *sz |= mbr[0x1be + (16 * part) + 13] << 8; *sz |= mbr[0x1be + (16 * part) + 14] << 16; *sz |= mbr[0x1be + (16 * part) + 15] << 24; } return ret; } static int setmbr(void) { unsigned char mbr[512]; unsigned int sz, offs; int ret, i; char var[32]; char *e; unsigned char *x; ret = dblstor_read(&cb, 0, mbr, 1); if (ret) return ret; //assert(is_cavium()); //if (is_cavium()) x = cavium_mbr; else x = ts735x_mbr; x = cavium_mbr; if (mbr[510] != 0x55 || mbr[511] != 0xaa) memcpy(mbr, x, 512); else memcpy(mbr, x, 446); for (i = 0; i < 4; i++) { sprintf(var, "part%d_offs", i+1); e = getenv(var); if (e) { offs = strtoul(e, NULL, 0); mbr[0x1be + (16 * i) + 8] = offs & 0xff; mbr[0x1be + (16 * i) + 9] |= (offs>>8) & 0xff; mbr[0x1be + (16 * i) + 10] |= (offs>>16) & 0xff; mbr[0x1be + (16 * i) + 11] |= offs>>24; } sprintf(var, "part%d_sz", i+1); e = getenv(var); if (e) { sz = strtoul(e, NULL, 0); mbr[0x1be + (16 * i) + 12] = sz & 0xff; mbr[0x1be + (16 * i) + 13] |= (sz>>8) & 0xff; mbr[0x1be + (16 * i) + 14] |= (sz>>16) & 0xff; mbr[0x1be + (16 * i) + 15] |= sz>>24; } sprintf(var, "part%d_type", i+1); e = getenv(var); if (e) mbr[0x1be + (16 * i) + 4] = strtoul(e, NULL, 0); } dblstor_write(&cb, 0, mbr, 1); dblstor_read(&cb, 0, mbr, 1); /* To ensure write is terminated */ return 0; } #ifdef __linux__ #define NBD_SET_SOCK _IO(0xab, 0) #define NBD_SET_BLKSIZE _IO(0xab, 1) #define NBD_DO_IT _IO(0xab, 3) #define NBD_CLEAR_SOCK _IO(0xab, 4) #define NBD_CLEAR_QUEUE _IO(0xab, 5) #define NBD_SET_SIZE_BLOCKS _IO(0xab, 7) static void nbdclient(int n, struct sockaddr_un *su) { int sk, len, fd, x; char buf[152]; unsigned long long llsz = (unsigned long long)cb.size * 512ULL; sprintf(buf, "/dev/nbd%d", n); fd = open(buf, O_RDWR); if (fd == -1) { sprintf(buf, "/dev/nbd/%d", n); fd = open(buf, O_RDWR); } assert (fd != -1); ioctl(fd, NBD_SET_BLKSIZE, 4096); ioctl(fd, NBD_SET_SIZE_BLOCKS, llsz / 4096); ioctl(fd, NBD_CLEAR_QUEUE); ioctl(fd, NBD_CLEAR_SOCK); sk = socket(PF_UNIX, SOCK_STREAM, 0); x = connect(sk, (struct sockaddr *)su, sizeof(struct sockaddr_un)); assert (x == 0); len = 152; while (len > 0) { x = TEMP_FAILURE_RETRY(read(sk, buf, len)); assert (x > 0); len -= x; } x = ioctl(fd, NBD_SET_SOCK, sk); assert (x == 0); ioctl(fd, NBD_DO_IT); ioctl(fd, NBD_CLEAR_QUEUE); ioctl(fd, NBD_CLEAR_SOCK); close(sk); close(fd); exit(0); } static void nbdclient_poke(void) { static int poked = 0; if (!poked) { int fd; char buf[1]; poked = 1; if (fork() == 0) { fd = open("/dev/nbd0", O_RDONLY); if (fd == -1) fd = open("/dev/nbd/0", O_RDONLY); read(fd, buf, 1); exit(0); } } } #else static void nbdclient(int n, struct sockaddr_un *su) { } #endif static void nbdserver(char *arg, int local, struct in_addr iface) { int i, n, ret, maxfd = 0, incomplete = 0; int infd[8], remfd[8]; int offs[8]; int numnbd; struct sockaddr_in si; struct sockaddr_un su; struct nbd_request req; struct nbd_reply resp; unsigned int reqmagic; struct timeval tv; struct itimerval it; struct sigaction siga; fd_set rfds; FD_ZERO(&rfds); nbdbufsz = 128 * 1024 / 512; nbdbuf = malloc(nbdbufsz * 512 + 4095); nbdbuf = (unsigned char *)(((size_t)nbdbuf + 4095) & ~0xfff); bzero(nbdbuf, nbdbufsz * 512); bzero(shm_dmesg->int32, sizeof(shm_dmesg->int32)); bzero(shm_dmesg->int64, sizeof(shm_dmesg->int64)); numnbd = 0; while(*arg) { if (*arg == ',') arg++; if (strncmp(arg, "lun", 3) != 0) break; arg += 3; if (!isdigit(*arg)) break; n = *arg - '0'; assert(n == 0); /* not yet ready for lun 1 */ arg++; if (*arg != ':') break; arg++; if (strncmp(arg, "part", 4) == 0) { arg += 4; if (!isdigit(*arg)) break; offs[numnbd] = -(*arg - '0'); arg++; } else if (strncmp(arg, "disc", 4) == 0) { arg += 4; offs[numnbd] = 0; } else break; numnbd++; } if (*arg != 0) { fprintf(stderr, "NBDSPEC parse error\n"); exit(9); } for (i = 0; i < numnbd; i++) { int r, sk, x = 1; if (local) { sk = socket(PF_UNIX, SOCK_STREAM, 0); assert(sk != -1); bzero(&su, sizeof(struct sockaddr_un)); su.sun_family = AF_UNIX; sprintf(su.sun_path, "/nbdserver%d", i); unlink(su.sun_path); r = bind(sk, (struct sockaddr *)&su, sizeof(su)); } else { sk = socket(PF_INET, SOCK_STREAM, 0); assert(sk != -1); bzero(&si, sizeof(struct sockaddr_in)); si.sin_family = AF_INET; si.sin_addr.s_addr = iface.s_addr; si.sin_port = htons(7550 + i); setsockopt(sk, SOL_SOCKET, SO_REUSEADDR, &x, 4); r = bind(sk, (struct sockaddr *)&si, sizeof(si)); } if (r) { perror("bind"); exit(1); } r = listen(sk, 5); assert(r != -1); if (local && fork() == 0) nbdclient(i, &su); r = fcntl(sk, F_SETFL, O_NONBLOCK); assert(r != -1); if (sk > maxfd) maxfd = sk; FD_SET(sk, &rfds); infd[i] = sk; remfd[i] = -1; } resp.magic[0] = 0x67; resp.magic[1] = 0x44; resp.magic[2] = 0x66; resp.magic[3] = 0x98; reqmagic = htonl(0x25609513); if (!opt_foreground && !local) daemon(0, 0); shm_dmesg->int64[NBD_READREQS] = 0; shm_dmesg->int64[NBD_READ_BLKS] = 0; shm_dmesg->int32[NBDPID] = getpid(); signal(SIGPIPE, SIG_IGN); siga.sa_handler = alarmsig; sigemptyset(&siga.sa_mask); siga.sa_flags = SA_RESTART; sigaction(SIGALRM, &siga, NULL); alarmsig(0); it.it_interval.tv_sec = it.it_value.tv_sec = 1; it.it_interval.tv_usec = it.it_value.tv_usec = 0; setitimer(ITIMER_REAL, &it, NULL); superloop_start: tv.tv_sec = 0; if (incomplete) tv.tv_usec = 0; else tv.tv_usec = 200000; ret = select(maxfd + 1, &rfds, NULL, NULL, &tv); if (ret == 0) { incomplete = nbdidle(); #ifdef __linux__ if (local && !opt_foreground) { if (remfd[0] != -1) nbdclient_poke(); if (shm_dmesg->int64[NBD_READREQS] > 0) { local = 0; daemon(0, 0); shm_dmesg->int32[NBDPID] = getpid(); signal(SIGPIPE, SIG_IGN); sigaction(SIGALRM, &siga, NULL); setitimer(ITIMER_REAL, &it, NULL); } } #endif } if (ret == -1 && errno == EINTR) FD_ZERO(&rfds); for (i = 0; i < numnbd; i++) { int r; if (remfd[i] == -1) continue; if (FD_ISSET(remfd[i], &rfds)) { unsigned int sec, len; req.magic = 0; r = xread(remfd[i], &req, sizeof(req)); if (r == 0) goto eof; assert (r == sizeof(req)); assert (req.magic == reqmagic); incomplete = resp.error = 0; /* lazy partition table parse */ if (offs[i] < 0 && !resp.error) { r = get_ptbl_offs(-offs[i], NULL, NULL); if (r > 0) offs[i] = r; else resp.error = htonl(1); } memcpy(resp.handle, req.handle, 8); len = ntohl(req.len); assert ((len & 0x1ff) == 0); len = len >> 9; sec = (ntohl(req.fromhi) & 0x1ff) << 23; sec |= ntohl(req.fromlo) >> 9; sec += offs[i]; if ((sec + len) > cb.size) { shm_dmesg->int32[NBD_SEEK_PAST_EOF_ERRS]++; resp.error = htonl(1); } if (resp.error) { if (req.type[3] == 1) xread(remfd[i], NULL, len << 9); r = xwrite(remfd[i], &resp, sizeof(resp)); assert (r == sizeof(resp)); continue; } if (req.type[3] == 0) { /* READ */ r = xwrite(remfd[i], &resp, sizeof(resp)); if (r == 0) goto eof; assert (r == sizeof(resp)); nbdwrite(remfd[i], sec, len); shm_dmesg->int64[NBD_READREQS]++; shm_dmesg->int64[NBD_READ_BLKS] += len; } else if (req.type[3] == 1) { /* WRITE */ nbdread(remfd[i], sec, len); shm_dmesg->int64[NBD_WRITEREQS]++; shm_dmesg->int64[NBD_WRITE_BLKS] += len; r = xwrite(remfd[i], &resp, sizeof(resp)); if (r == 0) goto eof; assert (r == sizeof(resp)); } else goto eof; } else FD_SET(remfd[i], &rfds); continue; eof: shutdown(remfd[i], SHUT_RDWR); FD_CLR(remfd[i], &rfds); close(remfd[i]); remfd[i] = -1; } for (i = 0; i < numnbd; i++) { /* new connection requests pending accept? */ if (FD_ISSET(infd[i], &rfds)) { int r, sk, tos = IPTOS_LOWDELAY; int x; unsigned long long llsz; char d[128] = {0x00, 0x00, 0x42, 0x02, 0x81, 0x86, 0x12, 0x53}; struct linger l; if (remfd[i] != -1) { shutdown(remfd[i], SHUT_RDWR); FD_CLR(remfd[i], &rfds); close(remfd[i]); remfd[i] = -1; } sk = accept(infd[i], NULL, 0); if (sk == -1) { FD_SET(infd[i], &rfds); continue; } setsockopt(sk, IPPROTO_IP, IP_TOS, &tos, 4); x = 1; setsockopt(sk, IPPROTO_TCP, TCP_NODELAY, &x, 4); setsockopt(sk, SOL_SOCKET, SO_KEEPALIVE, &x, 4); setsockopt(sk, SOL_SOCKET, SO_OOBINLINE, &x, 4); l.l_onoff = 0; l.l_linger = 0; setsockopt(sk, SOL_SOCKET, SO_LINGER, &l, sizeof(l)); if (sk > maxfd) maxfd = sk; FD_SET(sk, &rfds); remfd[i] = sk; r = xwrite(sk, "NBDMAGIC", 8); assert (r == 8); r = xwrite(sk, d, 8); assert (r == 8); /* size */ llsz = (unsigned long long)cb.size * 512ULL; d[0] = llsz >> 56; d[1] = llsz >> 48; d[2] = llsz >> 40; d[3] = llsz >> 32; d[4] = llsz >> 24; d[5] = llsz >> 16; d[6] = llsz >> 8; d[7] = llsz; r = xwrite(sk, d, 8); assert (r == 8); bzero(d, 128); r = xwrite(sk, d, 128); assert (r == 128); } else { FD_SET(infd[i], &rfds); } } goto superloop_start; } static int parse_time(char *s) { unsigned int n; char *b = s; n = strtoul(b, &b, 0); if (n == 0) return 0; if (*b == 'W' || *b == 'w') return n * 7 * 24 * 3600; else if (*b == 'd' || *b == 'D') return n * 24 * 3600; else if (*b == 'h' || *b == 'H') return n * 3600; else return 0; } #define I1E6 1000000ULL #define I1E7 10000000ULL #define I1E9 1000000000ULL #define I1E10 10000000000ULL #define I1E12 1000000000000ULL static unsigned long long parse_sz(char *s) { int state = 0, f; long long int n = 0, d = 0; char *b = s, *x; for (;;) switch(state) { case 0: /* building a number in n */ if (*b == '.') state = 2; else if (!isdigit(*b)) return -1; else { n = strtoll(b, &b, 0); state++; } break; case 1: if (*b == '.') state++; else if (*b == 0) { /* sectors */ return n; } else if (*b == 'b' || *b == 'B') { /* bytes */ return n / 512; } else if (*b == 'k' || *b == 'K') { /* KB */ return (n * 1000 + d) / 512; } else if (*b == 'm' || *b == 'M') { /* MB */ return (n * I1E6 + d * 1000) / 512; } else if (*b == 'g' || *b == 'G') { /* GB */ return (n * I1E9 + d * I1E6) / 512; } else if (*b == 't' || *b == 'T') { /* TB */ return (n * I1E12 + d * I1E9) / 512; } b++; break; case 2: /* Parse fraction */ if (!isdigit(*b)) return -1; x = b; d = strtoul(b, &b, 10); for (f = 1; x != b; x++) f *= 10; d = d * 1000 / f; /* d contains thousandths */ state = 1; break; } } static char *humanize_sz(unsigned long long sects) { static char a[4][16]; static char which; char *c = a[(which++ & 3)]; unsigned long long bytes = sects * 512; if (bytes < I1E6) sprintf(c, "%lldKB", bytes / 1000); else if (bytes < I1E9) sprintf(c, "%lld.%02lldMB", bytes / I1E6, bytes % I1E6 / 10000); else if (bytes < I1E12) sprintf(c, "%lld.%02lldGB", bytes / I1E9, bytes % I1E9 / I1E7); else sprintf(c, "%lld.%02lldTB", bytes / I1E12, bytes % I1E12 / I1E10); return c; } static void printk(struct dmesg *dm, const char *fmt, ...) { char buf[256], *b; va_list ap; int len, p = dm->put, n, g; va_start(ap, fmt); len = snprintf(buf, 32, "%x ", cb.curtime); len += vsnprintf(&buf[len], sizeof(buf)-len, fmt, ap); if (opt_foreground) fwrite(buf, len, 1, stderr); #ifndef __arm__ /* The __sync_*() functions are gcc extensions for atomic ops */ while (!__sync_bool_compare_and_swap(&dm->lock, 0, 1)) sched_yield(); #else for (;;) { n = 1; asm volatile("swpb %0, %1, [%2]" :"=&r" (n) : "r" (n), "r" (&dm->lock) : "memory"); if (n == 1) sched_yield(); else break; } #endif g = dm->get; n = (p - g) & 0x3fff; b = dm->buf; /* We're overflowing the buffer, clear a line */ while ((n + len) > 0x3ffe) { while(b[g] != '\n') g = (g + 1) & 0x3fff; g = (g + 1) & 0x3fff; n = (p - g) & 0x3fff; } b = buf; while (len--) { dm->buf[p++] = *b++; p &= 0x3fff; } dm->put = p; dm->get = g; asm volatile ("" : : : "memory"); dm->lock = 0; } void dmesg(struct dmesg *dm, int follow, FILE *out) { int i, p, g; char buf[16384], *b; again: i = 0; #ifndef __arm__ while (!__sync_bool_compare_and_swap(&dm->lock, 0, 1)) sched_yield(); #else for (;;) { p = 1; asm volatile("swpb %0, %1, [%2]" :"=&r" (p) : "r" (p), "r" (&dm->lock) : "memory"); if (p == 1) sched_yield(); else break; } #endif p = dm->put; g = dm->get; while (p != g) { buf[i++] = dm->buf[g]; g = (g + 1) & 0x3fff; } if (follow) dm->get = g; asm volatile ("" : : : "memory"); dm->lock = 0; /* fwrite(buf, i, 1, out); */ for (b = buf; b != &buf[i];) { time_t now = strtoul(b, &b, 16); struct tm *tm; char tbuf[32]; tm = localtime(&now); strftime(tbuf, 32, "%b %e %T ", tm); fprintf(out, "%s", tbuf); b++; do fputc(*b, out); while (*b++ != '\n'); } if (follow) { usleep(10000); goto again; } } static char *taintstatus(void) { static char a[80]; unsigned int t = cb.taintstatus; if (t == 0) return "OK"; sprintf(a, "%s%s%s%s", t & PRIMARY_FAILED ? "PRIMARY_FAILED " : "", t & PRIMARY_TAINTED ? "PRIMARY_TAINTED " : "", t & FALLBACK_FAILED ? "FALLBACK_FAILED " : "", t & FALLBACK_TAINTED ? "FALLBACK_TAINTED" : ""); return a; } static char *dbgstr(unsigned int code) { switch (code) { case BADBLOCK: return "BADBLOCK"; case BADBLOCK_RECOVERED: return "BADBLOCK_RECOVERED"; case FALLBACK_BADBLOCK: return "FALLBACK_BADBLOCK"; case FALLBACK_RECOVERED: return "FALLBACK_RECOVERED"; case BADBLOCK_UNRECOVERABLE: return "BADBLOCK_UNRECOVERABLE"; case PRIMARY_CONFLICT: return "PRIMARY_CONFLICT"; case FALLBACK_CONFLICT: return "FALLBACK_CONFLICT"; case CRCFAIL: return "CRCFAIL"; } return ""; } static void dblstor_debug(void *arg, unsigned int code, ...) { static unsigned int seeks, reads, writes, reqs; static unsigned int last_time, rdwr, next_s; static unsigned int err_type, err_beg, err_len, err_seq, err_time; static unsigned int resilver_start; static unsigned int hr, sixhr, day, week, tot; static unsigned int ctype, cbeg, clen, ctime1, ctime2; static unsigned int ncrcfails, crcf_time, crcfail_ncont, crcfail_last; unsigned int s, l, ts, ts2, e, a, f, x; unsigned char *b, *b2; char buf[256]; va_list ap; FILE *save; va_start(ap, code); switch (code) { case BADBLOCK: case BADBLOCK_RECOVERED: case FALLBACK_BADBLOCK: case FALLBACK_RECOVERED: case BADBLOCK_UNRECOVERABLE: s = va_arg(ap, unsigned int); /* sector */ b = va_arg(ap, unsigned char *); /* buffer */ l = va_arg(ap, unsigned int); /* len */ if (opt_savebadblocks) { sprintf(buf, "%d.%s.0x%x", err_seq++, dbgstr(code), s); save = fopen(buf, "w"); if (save != NULL) { fwrite(b, l * 512, 1, save); fclose(save); } opt_savebadblocks--; } if (code == err_type && err_len && s == (err_beg + err_len)) { err_len += l; break; } else if (err_len) { printk(arg, "%s: sector 0x%x, len %d\n", dbgstr(err_type), err_beg, err_len); } err_type = code; err_beg = s; err_len = l; err_time = cb.curtime; break; case PRIMARY_CONFLICT: case FALLBACK_CONFLICT: s = va_arg(ap, unsigned int); /* sector */ ts = va_arg(ap, unsigned int); /* timestamp1 */ b = va_arg(ap, unsigned char *); /* buf1 */ ts2 = va_arg(ap, unsigned int); /* timestamp2 */ b2 = va_arg(ap, unsigned char *); /* buf2 */ if (opt_savebadblocks) { sprintf(buf, "%d.%s.0x%x", err_seq++, dbgstr(code), s); save = fopen(buf, "w"); if (save != NULL) { fwrite(b, 576, 1, save); fwrite(b2, 576, 1, save); fclose(save); } opt_savebadblocks--; } if (ctype == code && clen && s == (cbeg + clen) && ctime1 == ts && ctime2 == ts2) { clen++; break; } else if (clen) { printk(arg, "%s: sect 0x%x, len %d, %d vs %d\n", dbgstr(ctype), cbeg, clen, ctime1, ctime2); } ctype = code; cbeg = s; clen = 1; ctime1 = ts; ctime2 = ts2; break; case RESILVER_SANITY_FAILED: s = va_arg(ap, unsigned int); /* sector */ printk(arg, "Sanity check failed! sector 0x%x\n", s); break; case RESILVER_STARTING: resilver_start = cb.curtime; tot = hr = sixhr = day = week = 0; printk(arg, "Resilver starting, status = %s\n", taintstatus()); break; case RESILVER_COMPLETE: if (clen) { printk(arg, "%s: sect 0x%x, len %d, %d vs %d\n", dbgstr(ctype), cbeg, clen, ctime1, ctime2); clen = 0; } printk(arg, "Resilver done, errors: %d unrecoverable: %d conflicts: %d\n", cb.errors, cb.unrecoverable_errors, cb.conflicts); printk(arg, "Ages, <1hr:%s <6hr:%s <1day:%s <1week:%s\n", humanize_sz(hr), humanize_sz(sixhr), humanize_sz(day), humanize_sz(week)); break; case RESILVER_RESTARTING: resilver_start = cb.curtime; tot = hr = sixhr = day = week = 0; printk(arg, "Finished first resilver, starting 2nd...\n"); break; case TAINTSTATUS_COMMITTED: printk(arg, "New taint status committed: %s\n", taintstatus()); break; case RESILVER_VISIT: b = va_arg(ap, unsigned char *); /* buffer */ if (clen) { printk(arg, "%s: sect 0x%x, len %d, %d vs %d\n", dbgstr(ctype), cbeg, clen, ctime1, ctime2); clen = 0; } ts = resilver_start - get32(&b[524]); if (ts < 3600) hr++; if (ts < 3600 * 6) sixhr++; if (ts < 3600 * 24) day++; if (ts < 3600 * 24 * 7) week++; if (((tot>>8) * 100 / (cb.size>>8)) != (((tot + 1)>>8) * 100 / (cb.size>>8))) { int pct = (tot>>8) * 100 / (cb.size>>8)+1; printk(arg, "Resilver %d%% done...\n", pct); shm_dmesg->int32[DBLSTOR_RESILVER_PCT_DONE] = pct; } tot++; break; case READ_REQUEST: case WRITE_REQUEST: s = va_arg(ap, unsigned int); /* sector */ b = va_arg(ap, unsigned char *); /* buffer */ l = va_arg(ap, unsigned int); /* len */ if (rdwr != code || s != next_s) seeks++; if (code == READ_REQUEST) reads += l; else writes += l; next_s = s + l; rdwr = code; if (reqs++ == 0) last_time = cb.curtime; if (cb.curtime == last_time) break; case IDLE: if (crcf_time && (crcf_time != cb.curtime)) { if (ncrcfails >= 5) printk(arg, "%d more %scontiguous CRCFAILS\n", ncrcfails - 5, crcfail_ncont ? "non":""); crcf_time = ncrcfails = crcfail_ncont = 0; } if (err_len && cb.curtime != err_time) { printk(arg, "%s: sector 0x%x, len %d\n", dbgstr(err_type), err_beg, err_len); err_len = 0; } if (reqs && cb.curtime != last_time) { printk(arg, "reqs: %4d writes: %10s/%s " "reads: %10s seeks: %4d errs: %d/%d status: %s\n", reqs, humanize_sz(writes), humanize_sz(cb.curseq), humanize_sz(reads), seeks, cb.errors, cb.unrecoverable_errors, taintstatus()); reqs = reads = writes = seeks = 0; } break; case CRCFAIL: s = va_arg(ap, unsigned int); /* sector */ e = va_arg(ap, unsigned int); /* expected crc*/ a = va_arg(ap, unsigned int); /* actual sector */ x = va_arg(ap, unsigned int); /* actual crc */ f = va_arg(ap, unsigned int); /* doing fallback */ if (ncrcfails++ < 5) { printk(arg, "%s: sector 0x%x/%s, expected crc 0x%x, " "got crc:0x%x/sector:0x%x fallback=%d\n", dbgstr(code), s, humanize_sz(s), e, x, a, f); crcfail_ncont = 0; } else if (crcfail_last != s - 1) crcfail_ncont = 1; crcfail_last = s; if (crcf_time == 0) crcf_time = cb.curtime; break; } va_end(ap); } /* Low-level read with read-ahead */ static int ra_ll_read(void *arg, unsigned int s, unsigned char *buf, int len) { #define RA_BUF_SZ 128 * 1024 static unsigned char *ra_buf = NULL; static unsigned int ra_sect, ra_len; static void *ra_arg; unsigned int clen; if (ra_buf == NULL) ra_buf = malloc(RA_BUF_SZ); if ((s + (RA_BUF_SZ / 512)) >= ll_size) return ll_read(arg, s, buf, len); if (arg != ra_arg || s < ra_sect || s >= (ra_sect + ra_len)) { ll_read(arg, s, ra_buf, RA_BUF_SZ / 512); ra_sect = s; ra_len = RA_BUF_SZ / 512; ra_arg = arg; } clen = ra_len - (s - ra_sect); if (clen > len) clen = len; memcpy(buf, &ra_buf[512 * (s - ra_sect)], clen * 512); buf += clen * 512; len -= clen; s += clen; if (len) ra_ll_read(arg, s, buf, len); return 0; } static int ll_read(void *arg, unsigned int sect, unsigned char *buf, int len) { unsigned int blen = len * 512; int fd, ret = 0; #ifdef __linux__ off64_t s = sect, *o; #else off_t s = sect, *o; #endif if (arg == NULL) { bzero(buf, len * 512); return 1; } fd = *(int *)arg; if (arg == &ll_fd) o = &ll_fd_offset; else o = &fb_fd_offset; dly_ll_write(arg, 0, NULL, 0); /* Flush delayed writes */ #ifdef __linux__ if (*o != (s * 512)) ret = lseek64(fd, s * 512, SEEK_SET); #else if (*o != (s * 512)) ret = lseek(fd, s * 512, SEEK_SET); #endif if (ret == -1) return -1; while (blen > 0) { int x = TEMP_FAILURE_RETRY(read(fd, buf, blen)); if (x == 0) { bzero(buf, 512); x = 512; lseek(fd, 512, SEEK_CUR); } else if (x < 0) { return -1; } buf += x; blen -= x; } *o = (s + len) * 512; return 0; } static int dly_ll_write(void *arg, unsigned int s, unsigned char *buf, int len) { #define DLY_LL_BUF_SZ 128 * 1024 static unsigned char *dly_buf = NULL; static unsigned int dly_sect, dly_len; static void *dly_arg; unsigned int n; if (dly_buf == NULL) dly_buf = malloc(DLY_LL_BUF_SZ); if (buf && dly_len == 0) { if (len >= (DLY_LL_BUF_SZ / 512)) return ll_write(arg, s, buf, len); dly_arg = arg; dly_len = len; dly_sect = s; memcpy(dly_buf, buf, len*512); } else if (buf && s == (dly_sect + dly_len) && arg == dly_arg) { if ((dly_len + len) > (DLY_LL_BUF_SZ / 512)) { n = (DLY_LL_BUF_SZ / 512) - dly_len; dly_ll_write(arg, s, buf, n); dly_ll_write(arg, 0, NULL, 0); s += n; len -= n; buf += (512 * n); dly_ll_write(arg, s, buf, len); } else { memcpy(&dly_buf[dly_len * 512], buf, len*512); dly_len += len; } } else if (dly_len) { ll_write(dly_arg, dly_sect, dly_buf, dly_len); dly_len = 0; if (buf) dly_ll_write(arg, s, buf, len); } return 0; } static int ll_write(void *arg, unsigned int sect, unsigned char *buf, int len) { unsigned int blen = len * 512; int fd = *(int *)arg, ret = 0; #ifdef __linux__ off64_t s = sect, *o; #else off_t s = sect, *o; #endif assert(arg != NULL); if (arg == &ll_fd) o = &ll_fd_offset; else o = &fb_fd_offset; #ifdef __linux__ if (*o != (s * 512)) ret = lseek64(fd, s * 512, SEEK_SET); #else if (*o != (s * 512)) ret = lseek(fd, s * 512, SEEK_SET); #endif if (ret == -1) return -1; while (blen > 0) { int x = TEMP_FAILURE_RETRY(write(fd, buf, blen)); if (x <= 0) return -1; buf += x; blen -= x; } *o = (s + len) * 512; return 0; } static int ll_nbdread(unsigned int sect, unsigned char *buf, int len) { int ret; ret = dblstor_read(&cb, sect, buf, len); return ret; } static int ll_nbdwrite(unsigned int sect, unsigned char *buf, int len) { int ret; ret = dblstor_write(&cb, sect, buf, len); return ret; } static int nbdidle(void) { return dblstor_tend(&cb); } static void usage(char **argv) { fprintf(stderr, "Usage: %s [OPTION] ...\n" "Technologic Systems DoubleStore image manipulation.\n" "\n" "General options:\n" " -p, --primary=FILE Use FILE as primary store\n" " -F, --fallback=FILE Use FILE as fallback store\n" " -v, --resilver Force a resilvering of the dataset\n" " -Z, --size=SZ Set dataset size to SZ (2.0G, 50M, etc)\n" " -C, --init Initialize and zero the dataset\n" " -T, --stresstest Start a stress test\n" " -S, --savebadblocks Save bad blocks to files in the CWD\n" " -n, --random=SEED Do random seeks for tests\n" " -B, --nbd[=NSPEC] Run NBD userspace block driver client/server\n" " -d, --nbdserver[=NSPEC] Run NBD userspace block driver server\n" " -R, --read=N Read N blocks to stdout\n" " -W, --write=N Write N blocks\n" " -i, --writeimg=FILE Use FILE as file to write\n" " -t, --writetest Run write speed test\n" " -r, --readtest Run read speed test\n" " -a, --read-ahead Turn on 128kbyte read-ahead\n" " -c, --write-combine Enables combining small writes together\n" " -P, --printmbr Print MBR and partition table\n" " -M, --setmbr Write MBR from environment variables\n" " -f, --foreground Run NBD server in foreground\n" " -z, --blocksize=SZ Use SZ bytes each read/write call\n" " -k, --seek=SECTOR Seek to 512b sector number SECTOR\n" " -I, --bind=IPADDR Bind NBD server to IPADDR\n" " -G, --dmesg Print latest debug messages\n" " -L, --dmesg-follow Follow/tail debug messages\n" " -Q, --stats Print stats and status\n" " -u, --untaint Clear any error conditions\n" "\n" "When running a NBD server, NSPEC is a comma separated list of\n" "devices and partitions for the NBD servers starting at port 7550.\n" "e.g. \"lun0:part1,lun0:disc\" corresponds to 2 NBD servers, one at port\n" "7550 serving the first partition of the dataset, and the other at TCP\n" "port 7551 serving the whole disc device of the dataset.\n", argv[0] ); } static void alarmsig(int x) { struct timeval tv; int ret; ticks++; if (!no_update_curtime) { ret = gettimeofday(&tv, NULL); assert(ret == 0); cb.curtime = tv.tv_sec; } shm_dmesg->int32[DBLSTOR_TAINTSTATUS] = cb.taintstatus; shm_dmesg->int32[DBLSTOR_ERRORS] = cb.errors; shm_dmesg->int32[DBLSTOR_UNRECOVERABLE_ERRORS]=cb.unrecoverable_errors; shm_dmesg->int32[DBLSTOR_CONFLICTS] = cb.conflicts; shm_dmesg->int64[DBLSTOR_CURSEQ] = cb.curseq; shm_dmesg->int32[DBLSTOR_SIZE] = cb.size; shm_dmesg->int32[DBLSTOR_FB_OFFSET] = cb.fb_offset; } static unsigned int get_max_size_capable(void) { unsigned long long llsz = 0, fbsz = 0; unsigned int sz = 0; #ifdef __linux__ ioctl(ll_fd, BLKGETSIZE, &sz); llsz = sz; if (fb_fd) { ioctl(fb_fd, BLKGETSIZE, &sz); fbsz = sz; } #else ioctl(ll_fd, DKIOCGETBLOCKSIZE, &sz); ioctl(ll_fd, DKIOCGETBLOCKCOUNT, &llsz); llsz *= sz / 512; if (fb_fd) { ioctl(fb_fd, DKIOCGETBLOCKSIZE, &sz); ioctl(fb_fd, DKIOCGETBLOCKCOUNT, &fbsz); fbsz *= sz / 512; } #endif if (fbsz > llsz) llsz = fbsz; if (llsz && fbsz) llsz = llsz / 9 * 8; else llsz = llsz / 9 * 4; if (llsz >= 0x800) llsz -= 0x800; llsz &= ~0x7ff; return (int)llsz; } int main(int argc, char **argv) { int ret, fd, i, j, c, partseek, shmid; int opt_blocksz = 1, opt_random = 0, opt_writecombine = 0; int opt_read = 0, opt_readblks = 0, opt_readahead = 0; int opt_seek = 0, opt_init = 0, opt_writeset = 0, opt_resilver = 0; int opt_nbdserver = 0, opt_printmbr = 0, opt_setmbr = 0; int opt_stresstest = 0, opt_write = 0, opt_writeblks = 0; int opt_readtest = 0, opt_writetest = 0, opt_dmesg = 0; int opt_stats = 0, opt_untaint = 0, opt_local = 0; unsigned int sizemask, randomseed = 0; char *opt_nbdarg = NULL; struct in_addr opt_bind = { INADDR_ANY }; struct itimerval it; struct sigaction sa; FILE *ifile = NULL; static struct option long_options[] = { { "primary", 1, 0, 'p' }, { "fallback", 1, 0, 'F' }, { "resilver", 0, 0, 'v' }, { "size", 1, 0, 'Z' }, { "init", 9, 0, 'C' }, { "stresstest", 0, 0, 'T' }, { "savebadblocks", 0, 0, 'S' }, { "setmbr", 0, 0, 'M' }, { "printmbr", 0, 0, 'P' }, { "stats", 0, 0, 'Q' }, { "bind", 1, 0, 'D' }, { "writetest", 0, 0, 't' }, { "writeset", 1, 0, 'x' }, { "writeimg", 1, 0, 'i' }, { "seek", 1, 0, 'k' }, { "readtest", 0, 0, 'r' }, { "read-ahead", 0, 0, 'a' }, { "write-combine", 0, 0, 'c' }, { "read", 1, 0, 'R' }, { "random", 1, 0, 'n' }, { "write", 1, 0, 'W' }, { "blocksize", 1, 0, 'z' }, { "nbdserver", 2, 0, 'd' }, { "nbd", 2, 0, 'B' }, { "foreground", 0, 0, 'f' }, { "dmesg", 0, 0, 'G' }, { "dmesg-follow", 0, 0, 'L' }, { "untaint", 0, 0, 'u' }, { "help", 0, 0, 'h' }, { 0, 0, 0, 0 } }; for (i = 0; i < 3; i++) if (fcntl(i, F_GETFD) == -1) dup2(i ? i - 1 : open("/dev/null", O_RDWR), i); while((c = getopt_long(argc, argv, "p:F:Z:z:sSIR:W:d::fhk:i:D:CMPn:x:avcQTLuB::", long_options, NULL)) != -1) { switch (c) { case 'u': opt_untaint = 1; break; case 'Q': opt_stats = 1; break; case 'G': opt_dmesg = 1; break; case 'L': opt_dmesg = 2; break; case 't': opt_writetest = 1; break; case 'r': opt_readtest = 1; break; case 'v': opt_resilver = 1; break; case 'c': opt_writecombine = 1; break; case 'a': opt_readahead = 1; break; case 'x': opt_writeset = strtoul(optarg, NULL, 0); break; case 'M': opt_setmbr = 1; break; case 'P': opt_printmbr = 1; break; case 'D': i = inet_aton(optarg, &opt_bind); if (i == 0) { fprintf(stderr, "Bad arg: %s\n", optarg); return 3; } break; case 'z': opt_blocksz = strtoul(optarg, NULL, 0) / 512; break; case 'f': opt_foreground = 1; break; case 'B': opt_local = 1; opt_nbdserver = 1; if (optarg) opt_nbdarg = strdup(optarg); else opt_nbdarg = "lun0:disc"; break; case 'd': opt_nbdserver = 1; if (optarg) opt_nbdarg = strdup(optarg); else opt_nbdarg = "lun0:disc"; break; case 'W': opt_write = 1; opt_writeblks = strtoul(optarg, NULL, 0); break; case 'R': opt_read = 1; opt_readblks = strtoul(optarg, NULL, 0); break; case 'k': if (strcmp(optarg, "kernel") == 0) opt_seek = -32; else if (strcmp(optarg, "initrd") == 0) opt_seek = -33; else if (optarg[0] == 'p' && optarg[1] == 'a' && optarg[2] == 'r' && optarg[3] == 't') opt_seek = -1 * (optarg[4] - '0'); else opt_seek = strtoul(optarg, NULL, 0); break; case 'i': i = strlen(optarg); if (strcmp("-", optarg) == 0) ifile = stdin; else if (strcmp(".bz2", &optarg[i - 4]) == 0) { char b[512]; snprintf(b, 512, "exec bunzip2 -c '%s'", optarg); ifile = popen(b, "r"); } else ifile = fopen(optarg, "r"); if (!ifile) { perror(optarg); return 3; } if (opt_writeblks == 0) opt_writeblks = INT_MAX; opt_write = 1; break; case 'F': #ifdef __linux__ fd = open(optarg, O_RDWR|O_CREAT|O_LARGEFILE, 0666); #else fd = open(optarg, O_RDWR|O_CREAT, 0666); #endif if (fd == -1) { perror(optarg); return 3; } fb_fd = fd; cb.fb_arg = &fb_fd; break; case 'p': #ifdef __linux__ fd = open(optarg, O_RDWR|O_CREAT|O_LARGEFILE, 0666); #else fd = open(optarg, O_RDWR|O_CREAT, 0666); #endif if (fd == -1) { perror(optarg); return 3; } ll_fd = fd; cb.ll_arg = &ll_fd; break; case 'C': opt_init = 1; break; case 'Z': cb.size = (unsigned int)parse_sz(optarg); cb.size &= ~0x7ff; break; case 'T': opt_readahead = opt_writecombine = opt_stresstest = 1; break; case 'S': opt_savebadblocks = 1000; break; case 'n': randomseed = strtoul(optarg, NULL, 0); fprintf(stderr, "randomseed=0x%x\n", randomseed); opt_random = 1; break; case 'h': default: usage(argv); return 1; } } shmid = shmget(0x75200000, 0x5000, IPC_CREAT|0666); assert(shmid != -1); shm_dmesg = shmat(shmid, NULL, 0); assert(shm_dmesg != MAP_FAILED); if (opt_dmesg == 2) dmesg(shm_dmesg, 1, stderr); else if (opt_dmesg == 1) { dmesg(shm_dmesg, 0, stderr); return 0; } if (getpriority(PRIO_PROCESS, shm_dmesg->int32[NBDPID]) == -1 && errno == ESRCH) { shm_dmesg->int32[NBDPID] = 0; shm_dmesg->lock = 0; } if (opt_stats) { unsigned int *i32 = shm_dmesg->int32; unsigned long long *i64 = shm_dmesg->int64; fprintf(stderr, "nbdpid=%d\n" "nbd_readreqs=%lld\n" "nbd_read_blks=%lld\n" "humanized_nbd_read_blks=%s\n" "nbd_writereqs=%lld\n" "nbd_write_blks=%lld\n" "humanized_nbd_write_blks=%s\n" "nbd_seek_past_eof_errs=%d\n" "size=0x%x\n" "humanized_size=%s\n" "fb_offset=%d\n" "primary_tainted=%d\n" "primary_failed=%d\n" "fallback_tainted=%d\n" "fallback_failed=%d\n" "resilver_pct_done=%d\n" "lifetime_write_blks=%lld\n" "humanized_lifetime_write_blks=%s\n" "errors=%d\n" "unrecoverable_errors=%d\n" "conflicts=%d\n" "fallback_configuration=\"%s\"\n", i32[NBDPID], i64[NBD_READREQS], i64[NBD_READ_BLKS], humanize_sz(i64[NBD_READ_BLKS]), i64[NBD_WRITEREQS], i64[NBD_WRITE_BLKS], humanize_sz(i64[NBD_WRITE_BLKS]), i32[NBD_SEEK_PAST_EOF_ERRS], i32[DBLSTOR_SIZE], humanize_sz(i32[DBLSTOR_SIZE]), (int)i32[DBLSTOR_FB_OFFSET], (i32[DBLSTOR_TAINTSTATUS] & PRIMARY_TAINTED) ? 1 : 0, (i32[DBLSTOR_TAINTSTATUS] & PRIMARY_FAILED) ? 1 : 0, (i32[DBLSTOR_TAINTSTATUS] & FALLBACK_TAINTED) ? 1 : 0, (i32[DBLSTOR_TAINTSTATUS] & FALLBACK_FAILED) ? 1 : 0, i32[DBLSTOR_RESILVER_PCT_DONE], i64[DBLSTOR_CURSEQ], humanize_sz(i64[DBLSTOR_CURSEQ]), i32[DBLSTOR_ERRORS], i32[DBLSTOR_UNRECOVERABLE_ERRORS], i32[DBLSTOR_CONFLICTS], (int)i32[DBLSTOR_FB_OFFSET]<0 ? "separate disk" : "same disk" ); return 0; } if (ll_fd == 0) { fprintf(stderr, "Must specify --primary file\n"); return 1; } sa.sa_handler = alarmsig; sigemptyset(&sa.sa_mask); sa.sa_flags = SA_RESTART; sigaction(SIGALRM, &sa, NULL); alarmsig(0); it.it_interval.tv_sec = it.it_value.tv_sec = 1; it.it_interval.tv_usec = it.it_value.tv_usec = 0; setitimer(ITIMER_REAL, &it, NULL); if (opt_readahead) cb.ll_read = ra_ll_read; else cb.ll_read = ll_read; cb.debug = dblstor_debug; cb.debug_arg = shm_dmesg; printk(shm_dmesg, "dblstorctl starting...\n"); if (opt_init) { cb.resilver_state = 0xffffffff; if (cb.size == 0) cb.size = get_max_size_capable(); if (cb.fb_arg) cb.fb_offset = -cb.size; else cb.fb_offset = cb.size; if (cb.size == 0) { fprintf(stderr, "Must specify --size with --init\n"); return 1; } } else { unsigned char bigbuf[512]; ret = dblstor_read(&cb, 0, bigbuf, 1); fprintf(stderr, "autodetect_ok=%d\n", ret ? 0 : 1); if (ret && (opt_init||opt_stresstest||opt_write||opt_setmbr)) { opt_init = 1; cb.size = get_max_size_capable(); if (!cb.size) return 4; if (cb.fb_arg) cb.fb_offset = -cb.size; else cb.fb_offset = cb.size; cb.resilver_state = 0xffffffff; cb.taintstatus= cb.errors= cb.unrecoverable_errors= 0; } else if (ret) return 4; } if (opt_writecombine) cb.ll_write = dly_ll_write; else cb.ll_write = ll_write; fprintf(stderr, "size=0x%x\n", cb.size); fprintf(stderr, "humanized_size=%s\n", humanize_sz(cb.size)); assert((cb.size & 0x7) == 0); sizemask = 0; for (i = 0; i < 32; i++) { if (cb.size > (1 << (i + 1))) sizemask = (sizemask << 1) | 0x1; else break; } ll_size = cb.size / 8 * 9; if (cb.fb_offset > 0) ll_size *= 2; fprintf(stderr, "fb_offset=%d\n", cb.fb_offset); fprintf(stderr, "ll_minsize=0x%x\nhumanized_ll_minsize=%s\n", ll_size, humanize_sz(ll_size)); fprintf(stderr, "primary_tainted=%d\n", (cb.taintstatus & PRIMARY_TAINTED) ? 1 : 0); fprintf(stderr, "primary_failed=%d\n", (cb.taintstatus & PRIMARY_FAILED) ? 1 : 0); fprintf(stderr, "fallback_tainted=%d\n", (cb.taintstatus & FALLBACK_TAINTED) ? 1 : 0); fprintf(stderr, "fallback_failed=%d\n", (cb.taintstatus & FALLBACK_FAILED) ? 1 : 0); if (opt_init) { unsigned int bigbuf[512 * 8 / sizeof(unsigned int)]; bzero(bigbuf, sizeof(bigbuf)); no_update_curtime = 1; cb.curtime = 0; for (i = 0; i < cb.size; i += 8) dblstor_write(&cb, i, (unsigned char *)bigbuf, 8); dblstor_tend(&cb); dblstor_read(&cb, 0, (unsigned char *)bigbuf, 1); cb.resilver_state = 0; fprintf(stderr, "init_ok=1\n"); no_update_curtime = 0; } if (opt_resilver) { cb.resilver_state = 0; while (cb.resilver_state != 0xffffffff) dblstor_tend(&cb); fprintf(stderr, "resilver_done=1\n" "primary_tainted=%d\n" "primary_failed=%d\n" "fallback_tainted=%d\n" "fallback_failed=%d\n" "lifetime_write_blks=0x%llx\n" "humanized_lifetime_write_blks=\"%s\"\n" "errors=%d\n" "unrecoverable_errors=%d\n" "conflicts=%d\n" "fallback_configuration=\"%s\"\n", (cb.taintstatus & PRIMARY_TAINTED) ? 1 : 0, (cb.taintstatus & PRIMARY_FAILED) ? 1 : 0, (cb.taintstatus & FALLBACK_TAINTED) ? 1 : 0, (cb.taintstatus & FALLBACK_FAILED) ? 1 : 0, cb.curseq, humanize_sz(cb.curseq), cb.errors, cb.unrecoverable_errors, cb.conflicts, cb.fb_offset < 0 ? "separate disk" : "same disk"); } if (opt_printmbr) { int ty, sz, offs, hasext; for (i = 0, hasext = 0; i < 4; i++) { offs = get_ptbl_offs(i+1, &ty, &sz); fprintf(stderr, "part%d_offs=0x%x\n", i+1, offs); fprintf(stderr, "part%d_sz=0x%x\n", i+1, sz); fprintf(stderr, "part%d_type=0x%x\n", i+1, ty); if (ty == 5) { fprintf(stderr, "has_extended_partition=1\n"); hasext = 1; } } i = 4; if (hasext) while ((offs = get_ptbl_offs(i+1, &ty, &sz)) != -1) { fprintf(stderr, "part%d_offs=0x%x\n", i+1, offs); fprintf(stderr, "part%d_sz=0x%x\n", i+1, sz); fprintf(stderr, "part%d_type=0x%x\n", i+1, ty); i++; } } if (opt_setmbr) { unsigned char bigbuf[512]; i = setmbr(); dblstor_tend(&cb); dblstor_read(&cb, 0, (unsigned char *)bigbuf, 1); fprintf(stderr, "setmbr_ok=%d\n", i ? 0 : 1); } if (opt_seek == -32) { /* kernel partition */ int ty; partseek = 1; for (i = 0; i < 4; i++) { opt_seek = get_ptbl_offs(i, &ty, NULL); if (ty == 0xda) break; } } else if (opt_seek == -33) { /* initrd partition */ int ty, np = 0; partseek = 1; for (i = 0; i < 4; i++) { opt_seek = get_ptbl_offs(i, &ty, NULL); if (ty == 0xda) np++; if (np == 2) break; } } else if (opt_seek < 0) { partseek = 1; opt_seek = get_ptbl_offs(-opt_seek, NULL, NULL); } else partseek = 0; if (opt_read) { unsigned char *buf, *bufpg; struct timeval tv1, tv2; unsigned long long ll; buf = malloc(opt_blocksz * 512 + 4095); bzero(buf, opt_blocksz * 512); bufpg = (unsigned char *)(((size_t)buf + 4095) & ~0xfff); srandom(randomseed); if (opt_readblks > (cb.size/opt_blocksz) || opt_readblks == -1) opt_readblks = (cb.size/opt_blocksz); gettimeofday(&tv1, NULL); for (i = 0, j = opt_seek; i < opt_readblks; i++) { if ((j + opt_blocksz) > cb.size) { ret = dblstor_read(&cb, j, bufpg, cb.size - j); } else ret = dblstor_read(&cb, j, bufpg, opt_blocksz); assert(ret == 0); ret = fwrite(bufpg, opt_blocksz * 512, 1, stdout); if (ret != 1) break; if (opt_random) j = (unsigned int)random() & sizemask; else j += opt_blocksz; } gettimeofday(&tv2, NULL); j = (tv2.tv_sec - tv1.tv_sec) * 1000; j += tv2.tv_usec / 1000 - tv1.tv_usec / 1000; if (j <= 0) j = 1; i = i * opt_blocksz; ll = i * 512ULL; fprintf(stderr, "read_kb=%lld\n", ll / 1000); fprintf(stderr, "read_kbps=%lld\n", ll / j); free(buf); } if (opt_write) { unsigned char *buf, *bufpg; struct timeval tv1, tv2; int errs=0; unsigned long long ll; buf = malloc(opt_blocksz * 512 + 4095); bufpg = (unsigned char *)(((size_t)buf + 4095) & ~0xfff); memset(bufpg, opt_writeset, opt_blocksz * 512); srandom(randomseed); if (opt_writeblks>(cb.size/opt_blocksz) || opt_writeblks == -1) opt_writeblks = (cb.size/opt_blocksz); gettimeofday(&tv1, NULL); if (opt_seek && ifile && !partseek) { ret = fseek(ifile, opt_seek * 512, SEEK_SET); if (ret == -1) for (i = 0; i < opt_seek; i++) { fread(bufpg, 512, 1, ifile); } } for (i = 0, j = opt_seek; i < opt_writeblks; i++) { if (ifile) { ret = fread(bufpg, opt_blocksz * 512, 1, ifile); if (ret == 0 && feof(ifile)) break; assert(ret == 1); } if ((j + opt_blocksz) > cb.size) { ret = dblstor_write(&cb, j, bufpg, cb.size-j); } else ret = dblstor_write(&cb, j, bufpg, opt_blocksz); if (ret != 0) { fprintf(stderr, "error%d=%d\n", errs++, j); ret = dblstor_write(&cb, j, bufpg, opt_blocksz); } if (ret != 0) { fprintf(stderr, "perm_write_error=1\n"); exit(1); } if (opt_random) j = (unsigned int)random() & sizemask; else j += opt_blocksz; } gettimeofday(&tv2, NULL); j = (tv2.tv_sec - tv1.tv_sec) * 1000; j += tv2.tv_usec / 1000 - tv1.tv_usec / 1000; if (j <= 0) j = 1; i = i * opt_blocksz; ll = i * 512ULL; fprintf(stderr, "write_kb=%lld\n", ll / 1000); fprintf(stderr, "write_kbps=%lld\n", ll / j); /* This makes sure previous writes are committed */ dblstor_tend(&cb); dblstor_read(&cb, 0, (unsigned char *)buf, 1); free(buf); } if (opt_writetest) { time_t start; unsigned char *buf, *bufpg; buf = malloc(opt_blocksz * 512 + 4095); bufpg = (unsigned char *)(((size_t)buf + 4095) & ~0xfff); memset(bufpg, opt_writeset, opt_blocksz * 512); srandom(randomseed); fprintf(stderr, "write_byte=0x%x\n", opt_writeset); i = 0; j = 0; /* Synchronize to the next second transition */ start = time(NULL); while (start == time(NULL)); start = time(NULL); while (time(NULL) - start < 10) { ret = dblstor_write(&cb, j, bufpg, opt_blocksz); assert(ret == 0); i += opt_blocksz; if (opt_random) j = random() & sizemask; else j += opt_blocksz; } /* This makes sure previous writes are committed */ cb.resilver_state = 0xffffffff; dblstor_tend(&cb); ret = dblstor_read(&cb, 0, bufpg, opt_blocksz); assert(ret == 0); fprintf(stderr, "writetest_kbps=%d\n", i * 512 / 10000); fprintf(stderr, "writetest_xfrps=%d\n", i / 10 / opt_blocksz); free(buf); } if (opt_readtest) { time_t start; unsigned char *buf, *bufpg; int l = opt_blocksz * 512; buf = malloc(l + 4095); bufpg = (unsigned char *)(((size_t)buf + 4095) & ~0xfff); srandom(randomseed); bzero(bufpg, l); i = 0; j = 0; /* Synchronize to the next second transition */ start = time(NULL); while (start == time(NULL)); start = time(NULL); while (time(NULL) - start < 10) { ret = dblstor_read(&cb, j, bufpg, opt_blocksz); assert(ret == 0); i += opt_blocksz; if (opt_random) j = random() & sizemask; else j += opt_blocksz; } fprintf(stderr, "readtest_kbps=%d\n", i * 512 / 10000); fprintf(stderr, "readtest_xfrps=%d\n", i / 10 / opt_blocksz); free(buf); } if (opt_stresstest) { int iter, j, k, z, done; unsigned char *buf; char *envlen; unsigned long long maxlen = 0; int tmlen = 0; envlen = getenv("STRESSTEST_LENGTH"); if (!envlen) /* do nothing */; else if ((z = parse_time(envlen)) != 0) tmlen = z; else if ((maxlen = parse_sz(envlen)) != 0) /* size arg */; else assert(0); while (cb.resilver_state != 0xffffffff) dblstor_tend(&cb); buf = malloc(128 * 1024); if (!cb.taintstatus) for(iter = z = done = 0;!done;iter++) { for (k = 0; (k <= z) && !done; k++) for (j = 0; j < (cb.size/(128*1024/512)); j++) { for (i = 0; i < (128*1024/4); i++) buf[i] = random(); dblstor_write(&cb, j * (128*1024/512), (unsigned char *)buf, 128*1024/512); if (maxlen && (cb.curseq > maxlen)) done = 1; if (tmlen && ticks >= tmlen) done = 1; if (done) break; } if (iter > 10) z = 8; cb.resilver_state = 0; while (cb.resilver_state != 0xffffffff) dblstor_tend(&cb); if (tmlen && ticks >= tmlen) done = 1; if (cb.taintstatus) done = 1; } fprintf(stderr, "stresstest_ok=%d\n", cb.taintstatus ? 0 : 1); fprintf(stderr, "dblstor_seq=%s\n", humanize_sz(cb.curseq)); fprintf(stderr, "ll_testsz=%s\n", humanize_sz(cb.curseq/8*18)); } if (opt_untaint) { cb.taintstatus = TAINTSTATUS_DIRTY; dblstor_tend(&cb); fprintf(stderr, "untaint_ok=1\n"); } if (opt_nbdserver) nbdserver(opt_nbdarg, opt_local, opt_bind); return 0; }