| /* |
| * Copyright (C) 2010-2012 by Dell Inc. All rights reserved. |
| * Copyright (C) 2011-2013 Red Hat, Inc. |
| * |
| * This file is released under the GPL. |
| * |
| * dm-switch is a device-mapper target that maps IO to underlying block |
| * devices efficiently when there are a large number of fixed-sized |
| * address regions but there is no simple pattern to allow for a compact |
| * mapping representation such as dm-stripe. |
| */ |
| |
| #include <linux/device-mapper.h> |
| |
| #include <linux/module.h> |
| #include <linux/init.h> |
| #include <linux/vmalloc.h> |
| |
| #define DM_MSG_PREFIX "switch" |
| |
| /* |
| * One region_table_slot_t holds <region_entries_per_slot> region table |
| * entries each of which is <region_table_entry_bits> in size. |
| */ |
| typedef unsigned long region_table_slot_t; |
| |
| /* |
| * A device with the offset to its start sector. |
| */ |
| struct switch_path { |
| struct dm_dev *dmdev; |
| sector_t start; |
| }; |
| |
| /* |
| * Context block for a dm switch device. |
| */ |
| struct switch_ctx { |
| struct dm_target *ti; |
| |
| unsigned nr_paths; /* Number of paths in path_list. */ |
| |
| unsigned region_size; /* Region size in 512-byte sectors */ |
| unsigned long nr_regions; /* Number of regions making up the device */ |
| signed char region_size_bits; /* log2 of region_size or -1 */ |
| |
| unsigned char region_table_entry_bits; /* Number of bits in one region table entry */ |
| unsigned char region_entries_per_slot; /* Number of entries in one region table slot */ |
| signed char region_entries_per_slot_bits; /* log2 of region_entries_per_slot or -1 */ |
| |
| region_table_slot_t *region_table; /* Region table */ |
| |
| /* |
| * Array of dm devices to switch between. |
| */ |
| struct switch_path path_list[0]; |
| }; |
| |
| static struct switch_ctx *alloc_switch_ctx(struct dm_target *ti, unsigned nr_paths, |
| unsigned region_size) |
| { |
| struct switch_ctx *sctx; |
| |
| sctx = kzalloc(struct_size(sctx, path_list, nr_paths), GFP_KERNEL); |
| if (!sctx) |
| return NULL; |
| |
| sctx->ti = ti; |
| sctx->region_size = region_size; |
| |
| ti->private = sctx; |
| |
| return sctx; |
| } |
| |
| static int alloc_region_table(struct dm_target *ti, unsigned nr_paths) |
| { |
| struct switch_ctx *sctx = ti->private; |
| sector_t nr_regions = ti->len; |
| sector_t nr_slots; |
| |
| if (!(sctx->region_size & (sctx->region_size - 1))) |
| sctx->region_size_bits = __ffs(sctx->region_size); |
| else |
| sctx->region_size_bits = -1; |
| |
| sctx->region_table_entry_bits = 1; |
| while (sctx->region_table_entry_bits < sizeof(region_table_slot_t) * 8 && |
| (region_table_slot_t)1 << sctx->region_table_entry_bits < nr_paths) |
| sctx->region_table_entry_bits++; |
| |
| sctx->region_entries_per_slot = (sizeof(region_table_slot_t) * 8) / sctx->region_table_entry_bits; |
| if (!(sctx->region_entries_per_slot & (sctx->region_entries_per_slot - 1))) |
| sctx->region_entries_per_slot_bits = __ffs(sctx->region_entries_per_slot); |
| else |
| sctx->region_entries_per_slot_bits = -1; |
| |
| if (sector_div(nr_regions, sctx->region_size)) |
| nr_regions++; |
| |
| if (nr_regions >= ULONG_MAX) { |
| ti->error = "Region table too large"; |
| return -EINVAL; |
| } |
| sctx->nr_regions = nr_regions; |
| |
| nr_slots = nr_regions; |
| if (sector_div(nr_slots, sctx->region_entries_per_slot)) |
| nr_slots++; |
| |
| if (nr_slots > ULONG_MAX / sizeof(region_table_slot_t)) { |
| ti->error = "Region table too large"; |
| return -EINVAL; |
| } |
| |
| sctx->region_table = vmalloc(array_size(nr_slots, |
| sizeof(region_table_slot_t))); |
| if (!sctx->region_table) { |
| ti->error = "Cannot allocate region table"; |
| return -ENOMEM; |
| } |
| |
| return 0; |
| } |
| |
| static void switch_get_position(struct switch_ctx *sctx, unsigned long region_nr, |
| unsigned long *region_index, unsigned *bit) |
| { |
| if (sctx->region_entries_per_slot_bits >= 0) { |
| *region_index = region_nr >> sctx->region_entries_per_slot_bits; |
| *bit = region_nr & (sctx->region_entries_per_slot - 1); |
| } else { |
| *region_index = region_nr / sctx->region_entries_per_slot; |
| *bit = region_nr % sctx->region_entries_per_slot; |
| } |
| |
| *bit *= sctx->region_table_entry_bits; |
| } |
| |
| static unsigned switch_region_table_read(struct switch_ctx *sctx, unsigned long region_nr) |
| { |
| unsigned long region_index; |
| unsigned bit; |
| |
| switch_get_position(sctx, region_nr, ®ion_index, &bit); |
| |
| return (READ_ONCE(sctx->region_table[region_index]) >> bit) & |
| ((1 << sctx->region_table_entry_bits) - 1); |
| } |
| |
| /* |
| * Find which path to use at given offset. |
| */ |
| static unsigned switch_get_path_nr(struct switch_ctx *sctx, sector_t offset) |
| { |
| unsigned path_nr; |
| sector_t p; |
| |
| p = offset; |
| if (sctx->region_size_bits >= 0) |
| p >>= sctx->region_size_bits; |
| else |
| sector_div(p, sctx->region_size); |
| |
| path_nr = switch_region_table_read(sctx, p); |
| |
| /* This can only happen if the processor uses non-atomic stores. */ |
| if (unlikely(path_nr >= sctx->nr_paths)) |
| path_nr = 0; |
| |
| return path_nr; |
| } |
| |
| static void switch_region_table_write(struct switch_ctx *sctx, unsigned long region_nr, |
| unsigned value) |
| { |
| unsigned long region_index; |
| unsigned bit; |
| region_table_slot_t pte; |
| |
| switch_get_position(sctx, region_nr, ®ion_index, &bit); |
| |
| pte = sctx->region_table[region_index]; |
| pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit); |
| pte |= (region_table_slot_t)value << bit; |
| sctx->region_table[region_index] = pte; |
| } |
| |
| /* |
| * Fill the region table with an initial round robin pattern. |
| */ |
| static void initialise_region_table(struct switch_ctx *sctx) |
| { |
| unsigned path_nr = 0; |
| unsigned long region_nr; |
| |
| for (region_nr = 0; region_nr < sctx->nr_regions; region_nr++) { |
| switch_region_table_write(sctx, region_nr, path_nr); |
| if (++path_nr >= sctx->nr_paths) |
| path_nr = 0; |
| } |
| } |
| |
| static int parse_path(struct dm_arg_set *as, struct dm_target *ti) |
| { |
| struct switch_ctx *sctx = ti->private; |
| unsigned long long start; |
| int r; |
| |
| r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), |
| &sctx->path_list[sctx->nr_paths].dmdev); |
| if (r) { |
| ti->error = "Device lookup failed"; |
| return r; |
| } |
| |
| if (kstrtoull(dm_shift_arg(as), 10, &start) || start != (sector_t)start) { |
| ti->error = "Invalid device starting offset"; |
| dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev); |
| return -EINVAL; |
| } |
| |
| sctx->path_list[sctx->nr_paths].start = start; |
| |
| sctx->nr_paths++; |
| |
| return 0; |
| } |
| |
| /* |
| * Destructor: Don't free the dm_target, just the ti->private data (if any). |
| */ |
| static void switch_dtr(struct dm_target *ti) |
| { |
| struct switch_ctx *sctx = ti->private; |
| |
| while (sctx->nr_paths--) |
| dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev); |
| |
| vfree(sctx->region_table); |
| kfree(sctx); |
| } |
| |
| /* |
| * Constructor arguments: |
| * <num_paths> <region_size> <num_optional_args> [<optional_args>...] |
| * [<dev_path> <offset>]+ |
| * |
| * Optional args are to allow for future extension: currently this |
| * parameter must be 0. |
| */ |
| static int switch_ctr(struct dm_target *ti, unsigned argc, char **argv) |
| { |
| static const struct dm_arg _args[] = { |
| {1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"}, |
| {1, UINT_MAX, "Invalid region size"}, |
| {0, 0, "Invalid number of optional args"}, |
| }; |
| |
| struct switch_ctx *sctx; |
| struct dm_arg_set as; |
| unsigned nr_paths, region_size, nr_optional_args; |
| int r; |
| |
| as.argc = argc; |
| as.argv = argv; |
| |
| r = dm_read_arg(_args, &as, &nr_paths, &ti->error); |
| if (r) |
| return -EINVAL; |
| |
| r = dm_read_arg(_args + 1, &as, ®ion_size, &ti->error); |
| if (r) |
| return r; |
| |
| r = dm_read_arg_group(_args + 2, &as, &nr_optional_args, &ti->error); |
| if (r) |
| return r; |
| /* parse optional arguments here, if we add any */ |
| |
| if (as.argc != nr_paths * 2) { |
| ti->error = "Incorrect number of path arguments"; |
| return -EINVAL; |
| } |
| |
| sctx = alloc_switch_ctx(ti, nr_paths, region_size); |
| if (!sctx) { |
| ti->error = "Cannot allocate redirection context"; |
| return -ENOMEM; |
| } |
| |
| r = dm_set_target_max_io_len(ti, region_size); |
| if (r) |
| goto error; |
| |
| while (as.argc) { |
| r = parse_path(&as, ti); |
| if (r) |
| goto error; |
| } |
| |
| r = alloc_region_table(ti, nr_paths); |
| if (r) |
| goto error; |
| |
| initialise_region_table(sctx); |
| |
| /* For UNMAP, sending the request down any path is sufficient */ |
| ti->num_discard_bios = 1; |
| |
| return 0; |
| |
| error: |
| switch_dtr(ti); |
| |
| return r; |
| } |
| |
| static int switch_map(struct dm_target *ti, struct bio *bio) |
| { |
| struct switch_ctx *sctx = ti->private; |
| sector_t offset = dm_target_offset(ti, bio->bi_iter.bi_sector); |
| unsigned path_nr = switch_get_path_nr(sctx, offset); |
| |
| bio_set_dev(bio, sctx->path_list[path_nr].dmdev->bdev); |
| bio->bi_iter.bi_sector = sctx->path_list[path_nr].start + offset; |
| |
| return DM_MAPIO_REMAPPED; |
| } |
| |
| /* |
| * We need to parse hex numbers in the message as quickly as possible. |
| * |
| * This table-based hex parser improves performance. |
| * It improves a time to load 1000000 entries compared to the condition-based |
| * parser. |
| * table-based parser condition-based parser |
| * PA-RISC 0.29s 0.31s |
| * Opteron 0.0495s 0.0498s |
| */ |
| static const unsigned char hex_table[256] = { |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, |
| 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
| 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 |
| }; |
| |
| static __always_inline unsigned long parse_hex(const char **string) |
| { |
| unsigned char d; |
| unsigned long r = 0; |
| |
| while ((d = hex_table[(unsigned char)**string]) < 16) { |
| r = (r << 4) | d; |
| (*string)++; |
| } |
| |
| return r; |
| } |
| |
| static int process_set_region_mappings(struct switch_ctx *sctx, |
| unsigned argc, char **argv) |
| { |
| unsigned i; |
| unsigned long region_index = 0; |
| |
| for (i = 1; i < argc; i++) { |
| unsigned long path_nr; |
| const char *string = argv[i]; |
| |
| if ((*string & 0xdf) == 'R') { |
| unsigned long cycle_length, num_write; |
| |
| string++; |
| if (unlikely(*string == ',')) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| cycle_length = parse_hex(&string); |
| if (unlikely(*string != ',')) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| string++; |
| if (unlikely(!*string)) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| num_write = parse_hex(&string); |
| if (unlikely(*string)) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| |
| if (unlikely(!cycle_length) || unlikely(cycle_length - 1 > region_index)) { |
| DMWARN("invalid set_region_mappings cycle length: %lu > %lu", |
| cycle_length - 1, region_index); |
| return -EINVAL; |
| } |
| if (unlikely(region_index + num_write < region_index) || |
| unlikely(region_index + num_write >= sctx->nr_regions)) { |
| DMWARN("invalid set_region_mappings region number: %lu + %lu >= %lu", |
| region_index, num_write, sctx->nr_regions); |
| return -EINVAL; |
| } |
| |
| while (num_write--) { |
| region_index++; |
| path_nr = switch_region_table_read(sctx, region_index - cycle_length); |
| switch_region_table_write(sctx, region_index, path_nr); |
| } |
| |
| continue; |
| } |
| |
| if (*string == ':') |
| region_index++; |
| else { |
| region_index = parse_hex(&string); |
| if (unlikely(*string != ':')) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| } |
| |
| string++; |
| if (unlikely(!*string)) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| |
| path_nr = parse_hex(&string); |
| if (unlikely(*string)) { |
| DMWARN("invalid set_region_mappings argument: '%s'", argv[i]); |
| return -EINVAL; |
| } |
| if (unlikely(region_index >= sctx->nr_regions)) { |
| DMWARN("invalid set_region_mappings region number: %lu >= %lu", region_index, sctx->nr_regions); |
| return -EINVAL; |
| } |
| if (unlikely(path_nr >= sctx->nr_paths)) { |
| DMWARN("invalid set_region_mappings device: %lu >= %u", path_nr, sctx->nr_paths); |
| return -EINVAL; |
| } |
| |
| switch_region_table_write(sctx, region_index, path_nr); |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Messages are processed one-at-a-time. |
| * |
| * Only set_region_mappings is supported. |
| */ |
| static int switch_message(struct dm_target *ti, unsigned argc, char **argv, |
| char *result, unsigned maxlen) |
| { |
| static DEFINE_MUTEX(message_mutex); |
| |
| struct switch_ctx *sctx = ti->private; |
| int r = -EINVAL; |
| |
| mutex_lock(&message_mutex); |
| |
| if (!strcasecmp(argv[0], "set_region_mappings")) |
| r = process_set_region_mappings(sctx, argc, argv); |
| else |
| DMWARN("Unrecognised message received."); |
| |
| mutex_unlock(&message_mutex); |
| |
| return r; |
| } |
| |
| static void switch_status(struct dm_target *ti, status_type_t type, |
| unsigned status_flags, char *result, unsigned maxlen) |
| { |
| struct switch_ctx *sctx = ti->private; |
| unsigned sz = 0; |
| int path_nr; |
| |
| switch (type) { |
| case STATUSTYPE_INFO: |
| result[0] = '\0'; |
| break; |
| |
| case STATUSTYPE_TABLE: |
| DMEMIT("%u %u 0", sctx->nr_paths, sctx->region_size); |
| for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++) |
| DMEMIT(" %s %llu", sctx->path_list[path_nr].dmdev->name, |
| (unsigned long long)sctx->path_list[path_nr].start); |
| break; |
| } |
| } |
| |
| /* |
| * Switch ioctl: |
| * |
| * Passthrough all ioctls to the path for sector 0 |
| */ |
| static int switch_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) |
| { |
| struct switch_ctx *sctx = ti->private; |
| unsigned path_nr; |
| |
| path_nr = switch_get_path_nr(sctx, 0); |
| |
| *bdev = sctx->path_list[path_nr].dmdev->bdev; |
| |
| /* |
| * Only pass ioctls through if the device sizes match exactly. |
| */ |
| if (ti->len + sctx->path_list[path_nr].start != |
| i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) |
| return 1; |
| return 0; |
| } |
| |
| static int switch_iterate_devices(struct dm_target *ti, |
| iterate_devices_callout_fn fn, void *data) |
| { |
| struct switch_ctx *sctx = ti->private; |
| int path_nr; |
| int r; |
| |
| for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++) { |
| r = fn(ti, sctx->path_list[path_nr].dmdev, |
| sctx->path_list[path_nr].start, ti->len, data); |
| if (r) |
| return r; |
| } |
| |
| return 0; |
| } |
| |
| static struct target_type switch_target = { |
| .name = "switch", |
| .version = {1, 1, 0}, |
| .module = THIS_MODULE, |
| .ctr = switch_ctr, |
| .dtr = switch_dtr, |
| .map = switch_map, |
| .message = switch_message, |
| .status = switch_status, |
| .prepare_ioctl = switch_prepare_ioctl, |
| .iterate_devices = switch_iterate_devices, |
| }; |
| |
| static int __init dm_switch_init(void) |
| { |
| int r; |
| |
| r = dm_register_target(&switch_target); |
| if (r < 0) |
| DMERR("dm_register_target() failed %d", r); |
| |
| return r; |
| } |
| |
| static void __exit dm_switch_exit(void) |
| { |
| dm_unregister_target(&switch_target); |
| } |
| |
| module_init(dm_switch_init); |
| module_exit(dm_switch_exit); |
| |
| MODULE_DESCRIPTION(DM_NAME " dynamic path switching target"); |
| MODULE_AUTHOR("Kevin D. O'Kelley <Kevin_OKelley@dell.com>"); |
| MODULE_AUTHOR("Narendran Ganapathy <Narendran_Ganapathy@dell.com>"); |
| MODULE_AUTHOR("Jim Ramsay <Jim_Ramsay@dell.com>"); |
| MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); |
| MODULE_LICENSE("GPL"); |