Skip to content

Commit 0ebf824

Browse files
SparkiDevSparkiDev
authored
Merge pull request #7117 from douzzer/20240105-linuxkm-commercial-POC
20240105-linuxkm-commercial-POC
2 parents 08aa39f + 9c36bb9 commit 0ebf824

3 files changed

Lines changed: 157 additions & 39 deletions

File tree

‎linuxkm/linuxkm_memory.c‎

Lines changed: 99 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,25 @@ struct wc_thread_fpu_count_ent {
8080
unsigned int fpu_state;
8181
};
8282
struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
83+
84+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
85+
86+
#pragma GCC diagnostic push
87+
#pragma GCC diagnostic ignored "-Wunused-parameter"
88+
#pragma GCC diagnostic ignored "-Wnested-externs"
89+
/* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */
90+
#undef CONFIG_X86_DEBUG_FPU
91+
#include "../kernel/fpu/internal.h"
92+
#include "../kernel/fpu/xstate.h"
93+
#pragma GCC diagnostic pop
94+
95+
static union wc_linuxkm_fpu_savebuf {
96+
byte buf[1024]; /* must be 64-byte-aligned */
97+
struct fpstate fpstate;
98+
} *wc_linuxkm_fpu_savebufs = NULL;
99+
100+
#endif /* WOLFSSL_COMMERCIAL_LICENSE */
101+
83102
#define WC_FPU_COUNT_MASK 0x7fffffffU
84103
#define WC_FPU_SAVED_MASK 0x80000000U
85104

@@ -111,7 +130,37 @@ WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
111130
return MEMORY_E;
112131
}
113132

114-
memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
133+
memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked
134+
* sizeof(wc_linuxkm_fpu_states[0]));
135+
136+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
137+
wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc(
138+
wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs));
139+
if (! wc_linuxkm_fpu_savebufs) {
140+
pr_err("allocation of %lu bytes for "
141+
"wc_linuxkm_fpu_savebufs failed.\n",
142+
WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
143+
* sizeof(*wc_linuxkm_fpu_savebufs));
144+
free(wc_linuxkm_fpu_states);
145+
wc_linuxkm_fpu_states = NULL;
146+
return MEMORY_E;
147+
}
148+
if ((uintptr_t)wc_linuxkm_fpu_savebufs
149+
& (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1))
150+
{
151+
pr_err("allocation of %lu bytes for "
152+
"wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n",
153+
WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
154+
* sizeof(*wc_linuxkm_fpu_savebufs),
155+
(uintptr_t)wc_linuxkm_fpu_savebufs);
156+
free(wc_linuxkm_fpu_savebufs);
157+
wc_linuxkm_fpu_savebufs = NULL;
158+
free(wc_linuxkm_fpu_states);
159+
wc_linuxkm_fpu_states = NULL;
160+
return MEMORY_E;
161+
}
162+
163+
#endif
115164

116165
return 0;
117166
}
@@ -141,11 +190,17 @@ void free_wolfcrypt_linuxkm_fpu_states(void) {
141190
}
142191
}
143192

193+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
194+
free(wc_linuxkm_fpu_savebufs);
195+
wc_linuxkm_fpu_savebufs = NULL;
196+
#endif
144197
free(wc_linuxkm_fpu_states);
145198
wc_linuxkm_fpu_states = NULL;
146199
}
147200

148-
/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */
201+
/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu
202+
* pushing/popping
203+
*/
149204
static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
150205
struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
151206
pid_t my_pid = task_pid_nr(current), i_pid;
@@ -194,7 +249,16 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p)
194249
}
195250
}
196251

197-
static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) {
252+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
253+
static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state(
254+
struct wc_thread_fpu_count_ent *state)
255+
{
256+
size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state);
257+
return &wc_linuxkm_fpu_savebufs[i].fpstate;
258+
}
259+
#endif
260+
261+
static void wc_linuxkm_fpu_state_release(struct wc_thread_fpu_count_ent *ent) {
198262
if (ent->fpu_state != 0) {
199263
static int warned_nonzero_fpu_state = 0;
200264
if (! warned_nonzero_fpu_state) {
@@ -228,25 +292,39 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void)
228292
}
229293

230294
if (irq_fpu_usable()) {
231-
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
232-
/* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */
295+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
296+
struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
297+
fpregs_lock();
298+
fpstate->xfeatures = ~0UL;
299+
os_xsave(fpstate);
300+
#else /* !WOLFSSL_COMMERCIAL_LICENSE */
301+
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
302+
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
303+
/* inhibit migration, which gums up the algorithm in
304+
* kernel_fpu_{begin,end}().
305+
*/
233306
migrate_disable();
234307
#endif
235308
kernel_fpu_begin();
236-
pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
309+
#endif /* !WOLFSSL_COMMERCIAL_LICENSE */
310+
/* set msb 0 to trigger kernel_fpu_end() at cleanup. */
311+
pstate->fpu_state = 1U;
237312
} else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
238313
static int warned_fpu_forbidden = 0;
239314
if (! warned_fpu_forbidden)
240315
pr_err("save_vector_registers_x86 called from IRQ handler.\n");
241-
wc_linuxkm_fpu_state_free(pstate);
316+
wc_linuxkm_fpu_state_release(pstate);
242317
return EPERM;
243318
} else {
244-
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
319+
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
320+
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
321+
!defined(WOLFSSL_COMMERCIAL_LICENSE)
245322
migrate_disable();
246323
#endif
247324
/* assume already safely in_kernel_fpu. */
325+
/* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
248326
pstate->fpu_state =
249-
WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
327+
WC_FPU_SAVED_MASK + 1U;
250328
}
251329

252330
return 0;
@@ -265,15 +343,23 @@ void restore_vector_registers_x86(void)
265343
return;
266344
}
267345

268-
if (pstate->fpu_state == 0U)
346+
if (pstate->fpu_state == 0U) {
347+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
348+
struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
349+
os_xrstor(fpstate, fpstate->xfeatures);
350+
fpregs_unlock();
351+
#else
269352
kernel_fpu_end();
270-
else
353+
#endif
354+
} else
271355
pstate->fpu_state = 0U;
272-
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
356+
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
357+
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
358+
!defined(WOLFSSL_COMMERCIAL_LICENSE)
273359
migrate_enable();
274360
#endif
275361

276-
wc_linuxkm_fpu_state_free(pstate);
362+
wc_linuxkm_fpu_state_release(pstate);
277363

278364
return;
279365
}

‎linuxkm/linuxkm_wc_port.h‎

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -333,23 +333,28 @@
333333
#endif
334334
typeof(nr_cpu_ids) *nr_cpu_ids;
335335

336-
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
336+
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
337337
/* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */
338338
typeof(migrate_disable) *migrate_disable;
339339
typeof(migrate_enable) *migrate_enable;
340340
#endif
341341

342342
#ifdef CONFIG_X86
343343
typeof(irq_fpu_usable) *irq_fpu_usable;
344-
/* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
345-
* released in kernel 5.11, backported to 5.4.93
346-
*/
347-
#ifdef kernel_fpu_begin
348-
typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
349-
#else
350-
typeof(kernel_fpu_begin) *kernel_fpu_begin;
351-
#endif
352-
typeof(kernel_fpu_end) *kernel_fpu_end;
344+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
345+
typeof(fpregs_lock) *fpregs_lock;
346+
typeof(fpregs_lock) *fpregs_unlock;
347+
#else /* !WOLFSSL_COMMERCIAL_LICENSE */
348+
/* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
349+
* released in kernel 5.11, backported to 5.4.93
350+
*/
351+
#ifdef kernel_fpu_begin
352+
typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
353+
#else
354+
typeof(kernel_fpu_begin) *kernel_fpu_begin;
355+
#endif
356+
typeof(kernel_fpu_end) *kernel_fpu_end;
357+
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
353358
#else /* !CONFIG_X86 */
354359
#error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
355360
#endif /* arch */
@@ -472,19 +477,24 @@
472477
#endif
473478
#define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
474479

475-
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
480+
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
476481
#define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable))
477482
#define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable))
478483
#endif
479484

480485
#ifdef CONFIG_X86
481486
#define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
482-
#ifdef kernel_fpu_begin
483-
#define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
484-
#else
485-
#define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
486-
#endif
487-
#define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
487+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
488+
#define fpregs_lock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_lock())
489+
#define fpregs_unlock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_unlock())
490+
#else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
491+
#ifdef kernel_fpu_begin
492+
#define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
493+
#else
494+
#define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
495+
#endif
496+
#define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
497+
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
488498
#else /* !CONFIG_X86 */
489499
#error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
490500
#endif /* archs */

‎linuxkm/module_hooks.c‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@
2020
*/
2121

2222
#ifndef WOLFSSL_LICENSE
23+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
24+
#define WOLFSSL_LICENSE "wolfSSL Commercial"
25+
#else
2326
#define WOLFSSL_LICENSE "GPL v2"
2427
#endif
28+
#endif
2529

2630
#define FIPS_NO_WRAPPERS
2731

@@ -346,6 +350,17 @@ static int my_preempt_count(void) {
346350
return preempt_count();
347351
}
348352

353+
/* ditto for fpregs_lock/fpregs_unlock */
354+
#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
355+
static void my_fpregs_lock(void) {
356+
fpregs_lock();
357+
}
358+
359+
static void my_fpregs_unlock(void) {
360+
fpregs_unlock();
361+
}
362+
#endif
363+
349364
static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
350365
memset(
351366
&wolfssl_linuxkm_pie_redirect_table,
@@ -442,21 +457,28 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
442457
#endif
443458
wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
444459

445-
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
460+
#if defined(CONFIG_SMP) && \
461+
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
462+
!defined(WOLFSSL_COMMERCIAL_LICENSE)
446463
wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable;
447464
wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable;
448465
#endif
449466

450467
#ifdef WOLFSSL_LINUXKM_SIMD_X86
451468
wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
452-
#ifdef kernel_fpu_begin
453-
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
454-
kernel_fpu_begin_mask;
455-
#else
456-
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
457-
kernel_fpu_begin;
458-
#endif
459-
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
469+
#ifdef WOLFSSL_COMMERCIAL_LICENSE
470+
wolfssl_linuxkm_pie_redirect_table.fpregs_lock = my_fpregs_lock;
471+
wolfssl_linuxkm_pie_redirect_table.fpregs_unlock = my_fpregs_unlock;
472+
#else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
473+
#ifdef kernel_fpu_begin
474+
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
475+
kernel_fpu_begin_mask;
476+
#else
477+
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
478+
kernel_fpu_begin;
479+
#endif
480+
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
481+
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
460482
#endif /* WOLFSSL_LINUXKM_SIMD_X86 */
461483

462484
#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */

0 commit comments

Comments
 (0)