From 3111a1ec558f658ce1c90dd31808946a814b0f03 Mon Sep 17 00:00:00 2001 From: hselasky Date: Mon, 4 Jan 2016 09:37:05 +0000 Subject: [PATCH] MFC r289563,r291481,r292537,r292538,r292542,r292543,r292544 and r292834: Update the LinuxKPI: - Add more functions and types. - Implement ACCESS_ONCE(), WRITE_ONCE() and READ_ONCE(). - Implement sleepable RCU mechanism using shared exclusive locks. - Minor workqueue cleanup: - Make some functions global instead of inline to ease debugging. - Fix some minor style issues. - In the zero delay case in queue_delayed_work() use the return value from taskqueue_enqueue() instead of reading "ta_pending" unlocked and also ensure the callout is stopped before proceeding. - Implement drain_workqueue() function. - Reduce memory consumption when allocating kobject strings in the LinuxKPI. Compute string length before allocating memory instead of using fixed size allocations. Make kobject_set_name_vargs() global instead of inline to save some bytes when compiling. Sponsored by: Mellanox Technologies git-svn-id: svn://svn.freebsd.org/base/stable/10@293151 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/ofed/include/linux/compiler.h | 26 +++++- sys/ofed/include/linux/file.h | 17 +++- sys/ofed/include/linux/kobject.h | 24 +---- sys/ofed/include/linux/linux_compat.c | 91 ++++++++++++++++++- sys/ofed/include/linux/srcu.h | 72 +++++++++++++++ sys/ofed/include/linux/types.h | 2 - sys/ofed/include/linux/workqueue.h | 121 ++++++++++++-------------- 7 files changed, 260 insertions(+), 93 deletions(-) create mode 100644 sys/ofed/include/linux/srcu.h diff --git a/sys/ofed/include/linux/compiler.h b/sys/ofed/include/linux/compiler.h index 9b1a5ad47..c7a3d28fe 100644 --- a/sys/ofed/include/linux/compiler.h +++ b/sys/ofed/include/linux/compiler.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -62,5 +63,28 @@ #define typeof(x) __typeof(x) #define uninitialized_var(x) x = x +#define __read_mostly __attribute__((__section__(".data.read_mostly"))) +#define __always_unused __unused +#define __must_check __result_use_check + +#define __printf(a,b) __printflike(a,b) + +#define barrier() __asm__ __volatile__("": : :"memory") + +#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) + +#define WRITE_ONCE(x,v) do { \ + barrier(); \ + ACCESS_ONCE(x) = (v); \ + barrier(); \ +} while (0) + +#define READ_ONCE(x) ({ \ + __typeof(x) __var; \ + barrier(); \ + __var = ACCESS_ONCE(x); \ + barrier(); \ + __var; \ +}) #endif /* _LINUX_COMPILER_H_ */ diff --git a/sys/ofed/include/linux/file.h b/sys/ofed/include/linux/file.h index b76a40865..1f7282236 100644 --- a/sys/ofed/include/linux/file.h +++ b/sys/ofed/include/linux/file.h @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -119,6 +119,21 @@ get_unused_fd(void) return fd; } +static inline int +get_unused_fd_flags(int flags) +{ + struct file *file; + int error; + int fd; + + error = falloc(curthread, &file, &fd, flags); + if (error) + return -error; + /* drop the extra reference */ + fdrop(file, curthread); + return fd; +} + static inline struct linux_file * alloc_file(int mode, const struct file_operations *fops) { diff --git a/sys/ofed/include/linux/kobject.h b/sys/ofed/include/linux/kobject.h index 159f07131..437d32f7a 100644 --- a/sys/ofed/include/linux/kobject.h +++ b/sys/ofed/include/linux/kobject.h @@ -87,29 +87,7 @@ kobject_get(struct kobject *kobj) return kobj; } -static inline int -kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) -{ - char *old; - char *name; - - old = kobj->name; - - if (old && !fmt) - return 0; - - name = kzalloc(MAXPATHLEN, GFP_KERNEL); - if (!name) - return -ENOMEM; - vsnprintf(name, MAXPATHLEN, fmt, args); - kobj->name = name; - kfree(old); - for (; *name != '\0'; name++) - if (*name == '/') - *name = '!'; - return (0); -} - +int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list); int kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...); diff --git a/sys/ofed/include/linux/linux_compat.c b/sys/ofed/include/linux/linux_compat.c index e9cc0c6fd..ef44c6df1 100644 --- a/sys/ofed/include/linux/linux_compat.c +++ b/sys/ofed/include/linux/linux_compat.c @@ -64,6 +64,8 @@ #include +#include + MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); #include @@ -90,7 +92,50 @@ panic_cmp(struct rb_node *one, struct rb_node *two) } RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); - + +int +kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) +{ + va_list tmp_va; + int len; + char *old; + char *name; + char dummy; + + old = kobj->name; + + if (old && fmt == NULL) + return (0); + + /* compute length of string */ + va_copy(tmp_va, args); + len = vsnprintf(&dummy, 0, fmt, tmp_va); + va_end(tmp_va); + + /* account for zero termination */ + len++; + + /* check for error */ + if (len < 1) + return (-EINVAL); + + /* allocate memory for string */ + name = kzalloc(len, GFP_KERNEL); + if (name == NULL) + return (-ENOMEM); + vsnprintf(name, len, fmt, args); + kobj->name = name; + + /* free old string */ + kfree(old); + + /* filter new string */ + for (; *name != '\0'; name++) + if (*name == '/') + *name = '!'; + return (0); +} + int kobject_set_name(struct kobject *kobj, const char *fmt, ...) { @@ -882,6 +927,50 @@ linux_completion_done(struct completion *c) return (isdone); } +void +linux_delayed_work_fn(void *arg) +{ + struct delayed_work *work; + + work = arg; + taskqueue_enqueue(work->work.taskqueue, &work->work.work_task); +} + +void +linux_work_fn(void *context, int pending) +{ + struct work_struct *work; + + work = context; + work->fn(work); +} + +void +linux_flush_fn(void *context, int pending) +{ +} + +struct workqueue_struct * +linux_create_workqueue_common(const char *name, int cpus) +{ + struct workqueue_struct *wq; + + wq = kmalloc(sizeof(*wq), M_WAITOK); + wq->taskqueue = taskqueue_create(name, M_WAITOK, + taskqueue_thread_enqueue, &wq->taskqueue); + atomic_set(&wq->draining, 0); + taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); + + return (wq); +} + +void +destroy_workqueue(struct workqueue_struct *wq) +{ + taskqueue_free(wq->taskqueue); + kfree(wq); +} + static void linux_compat_init(void *arg) { diff --git a/sys/ofed/include/linux/srcu.h b/sys/ofed/include/linux/srcu.h new file mode 100644 index 000000000..c20215b09 --- /dev/null +++ b/sys/ofed/include/linux/srcu.h @@ -0,0 +1,72 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SRCU_H_ +#define _LINUX_SRCU_H_ + +#include +#include +#include + +struct srcu_struct { + struct sx sx; +}; + +static inline int +init_srcu_struct(struct srcu_struct *srcu) +{ + sx_init(&srcu->sx, "SleepableRCU"); + return (0); +} + +static inline void +cleanup_srcu_struct(struct srcu_struct *srcu) +{ + sx_destroy(&srcu->sx); +} + +static inline int +srcu_read_lock(struct srcu_struct *srcu) +{ + sx_slock(&srcu->sx); + return (0); +} + +static inline void +srcu_read_unlock(struct srcu_struct *srcu, int key) +{ + sx_sunlock(&srcu->sx); +} + +static inline void +synchronize_srcu(struct srcu_struct *srcu) +{ + sx_xlock(&srcu->sx); + sx_xunlock(&srcu->sx); +} + +#endif /* _LINUX_SRCU_H_ */ diff --git a/sys/ofed/include/linux/types.h b/sys/ofed/include/linux/types.h index 9fff0ec91..b5def5583 100644 --- a/sys/ofed/include/linux/types.h +++ b/sys/ofed/include/linux/types.h @@ -36,8 +36,6 @@ #include #include -#define __read_mostly __attribute__((__section__(".data.read_mostly"))) - #ifndef __bitwise__ #ifdef __CHECKER__ #define __bitwise__ __attribute__((bitwise)) diff --git a/sys/ofed/include/linux/workqueue.h b/sys/ofed/include/linux/workqueue.h index 38cd2fedd..41ca80fc1 100644 --- a/sys/ofed/include/linux/workqueue.h +++ b/sys/ofed/include/linux/workqueue.h @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,10 +34,13 @@ #include #include +#include + #include struct workqueue_struct { struct taskqueue *taskqueue; + atomic_t draining; }; struct work_struct { @@ -46,11 +49,19 @@ struct work_struct { void (*fn)(struct work_struct *); }; +typedef __typeof(((struct work_struct *)0)->fn) work_func_t; + struct delayed_work { struct work_struct work; struct callout timer; }; +extern void linux_work_fn(void *, int); +extern void linux_flush_fn(void *, int); +extern void linux_delayed_work_fn(void *); +extern struct workqueue_struct *linux_create_workqueue_common(const char *, int); +extern void destroy_workqueue(struct workqueue_struct *); + static inline struct delayed_work * to_delayed_work(struct work_struct *work) { @@ -58,21 +69,11 @@ to_delayed_work(struct work_struct *work) return container_of(work, struct delayed_work, work); } - -static inline void -_work_fn(void *context, int pending) -{ - struct work_struct *work; - - work = context; - work->fn(work); -} - #define INIT_WORK(work, func) \ do { \ (work)->fn = (func); \ (work)->taskqueue = NULL; \ - TASK_INIT(&(work)->work_task, 0, _work_fn, (work)); \ + TASK_INIT(&(work)->work_task, 0, linux_work_fn, (work)); \ } while (0) #define INIT_DELAYED_WORK(_work, func) \ @@ -81,7 +82,7 @@ do { \ callout_init(&(_work)->timer, CALLOUT_MPSAFE); \ } while (0) -#define INIT_DEFERRABLE_WORK INIT_DELAYED_WORK +#define INIT_DEFERRABLE_WORK(...) INIT_DELAYED_WORK(__VA_ARGS__) #define schedule_work(work) \ do { \ @@ -91,20 +92,15 @@ do { \ #define flush_scheduled_work() flush_taskqueue(taskqueue_thread) -static inline int queue_work (struct workqueue_struct *q, struct work_struct *work) +static inline int +queue_work(struct workqueue_struct *wq, struct work_struct *work) { - (work)->taskqueue = (q)->taskqueue; - /* Return opposite val to align with Linux logic */ - return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task); -} - -static inline void -_delayed_work_fn(void *arg) -{ - struct delayed_work *work; - - work = arg; - taskqueue_enqueue(work->work.taskqueue, &work->work.work_task); + work->taskqueue = wq->taskqueue; + /* Check for draining */ + if (atomic_read(&wq->draining) != 0) + return (!work->work_task.ta_pending); + /* Return opposite value to align with Linux logic */ + return (!taskqueue_enqueue(wq->taskqueue, &work->work_task)); } static inline int @@ -113,57 +109,44 @@ queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work, { int pending; - pending = work->work.work_task.ta_pending; work->work.taskqueue = wq->taskqueue; - if (delay != 0) - callout_reset(&work->timer, delay, _delayed_work_fn, work); - else - _delayed_work_fn((void *)work); - + if (atomic_read(&wq->draining) != 0) { + pending = work->work.work_task.ta_pending; + } else if (delay != 0) { + pending = work->work.work_task.ta_pending; + callout_reset(&work->timer, delay, linux_delayed_work_fn, work); + } else { + callout_stop(&work->timer); + pending = taskqueue_enqueue(work->work.taskqueue, + &work->work.work_task); + } return (!pending); } -static inline bool schedule_delayed_work(struct delayed_work *dwork, - unsigned long delay) -{ - struct workqueue_struct wq; - wq.taskqueue = taskqueue_thread; - return queue_delayed_work(&wq, dwork, delay); -} - -static inline struct workqueue_struct * -_create_workqueue_common(char *name, int cpus) +static inline bool +schedule_delayed_work(struct delayed_work *dwork, + unsigned long delay) { - struct workqueue_struct *wq; + struct workqueue_struct wq; - wq = kmalloc(sizeof(*wq), M_WAITOK); - wq->taskqueue = taskqueue_create((name), M_WAITOK, - taskqueue_thread_enqueue, &wq->taskqueue); - taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); - - return (wq); + wq.taskqueue = taskqueue_thread; + atomic_set(&wq.draining, 0); + return (queue_delayed_work(&wq, dwork, delay)); } - #define create_singlethread_workqueue(name) \ - _create_workqueue_common(name, 1) + linux_create_workqueue_common(name, 1) #define create_workqueue(name) \ - _create_workqueue_common(name, MAXCPU) + linux_create_workqueue_common(name, MAXCPU) -static inline void -destroy_workqueue(struct workqueue_struct *wq) -{ - taskqueue_free(wq->taskqueue); - kfree(wq); -} +#define alloc_ordered_workqueue(name, flags) \ + linux_create_workqueue_common(name, 1) -#define flush_workqueue(wq) flush_taskqueue((wq)->taskqueue) +#define alloc_workqueue(name, flags, max_active) \ + linux_create_workqueue_common(name, max_active) -static inline void -_flush_fn(void *context, int pending) -{ -} +#define flush_workqueue(wq) flush_taskqueue((wq)->taskqueue) static inline void flush_taskqueue(struct taskqueue *tq) @@ -171,12 +154,20 @@ flush_taskqueue(struct taskqueue *tq) struct task flushtask; PHOLD(curproc); - TASK_INIT(&flushtask, 0, _flush_fn, NULL); + TASK_INIT(&flushtask, 0, linux_flush_fn, NULL); taskqueue_enqueue(tq, &flushtask); taskqueue_drain(tq, &flushtask); PRELE(curproc); } +static inline void +drain_workqueue(struct workqueue_struct *wq) +{ + atomic_inc(&wq->draining); + flush_taskqueue(wq->taskqueue); + atomic_dec(&wq->draining); +} + static inline int cancel_work_sync(struct work_struct *work) { @@ -213,7 +204,7 @@ cancel_delayed_work_sync(struct delayed_work *work) static inline bool mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork, - unsigned long delay) + unsigned long delay) { cancel_delayed_work(dwork); queue_delayed_work(wq, dwork, delay); -- 2.45.0