1 /* SPDX-License-Identifier: GPL-2.0 */ !! 1 #ifndef __ASM_PERCPU_H 2 #ifndef _ASM_GENERIC_PERCPU_H_ !! 2 #define __ASM_PERCPU_H 3 #define _ASM_GENERIC_PERCPU_H_ << 4 3 5 #include <linux/compiler.h> !! 4 #include <asm-generic/percpu.h> 6 #include <linux/threads.h> << 7 #include <linux/percpu-defs.h> << 8 5 9 #ifdef CONFIG_SMP !! 6 #endif /* __ASM_PERCPU_H */ 10 << 11 /* << 12 * per_cpu_offset() is the offset that has to << 13 * percpu variable to get to the instance for << 14 * << 15 * Most arches use the __per_cpu_offset array << 16 * some arches have their own ways of determin << 17 */ << 18 #ifndef __per_cpu_offset << 19 extern unsigned long __per_cpu_offset[NR_CPUS] << 20 << 21 #define per_cpu_offset(x) (__per_cpu_offset[x] << 22 #endif << 23 << 24 /* << 25 * Determine the offset for the currently acti << 26 * An arch may define __my_cpu_offset to provi << 27 * means of obtaining the offset to the per cp << 28 * current processor. << 29 */ << 30 #ifndef __my_cpu_offset << 31 #define __my_cpu_offset per_cpu_offset(raw_smp << 32 #endif << 33 #ifdef CONFIG_DEBUG_PREEMPT << 34 #define my_cpu_offset per_cpu_offset(smp_proce << 35 #else << 36 #define my_cpu_offset __my_cpu_offset << 37 #endif << 38 << 39 /* << 40 * Arch may define arch_raw_cpu_ptr() to provi << 41 * translations for raw_cpu_ptr(). << 42 */ << 43 #ifndef arch_raw_cpu_ptr << 44 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR << 45 #endif << 46 << 47 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA << 48 extern void setup_per_cpu_areas(void); << 49 #endif << 50 << 51 #endif /* SMP */ << 52 << 53 #ifndef PER_CPU_BASE_SECTION << 54 #ifdef CONFIG_SMP << 55 #define PER_CPU_BASE_SECTION ".data..percpu" << 56 #else << 57 #define PER_CPU_BASE_SECTION ".data" << 58 #endif << 59 #endif << 60 << 61 #ifndef PER_CPU_ATTRIBUTES << 62 #define PER_CPU_ATTRIBUTES << 63 #endif << 64 << 65 #define raw_cpu_generic_read(pcp) << 66 ({ << 67 *raw_cpu_ptr(&(pcp)); << 68 }) << 69 << 70 #define raw_cpu_generic_to_op(pcp, val, op) << 71 do { << 72 *raw_cpu_ptr(&(pcp)) op val; << 73 } while (0) << 74 << 75 #define raw_cpu_generic_add_return(pcp, val) << 76 ({ << 77 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)) << 78 << 79 *__p += val; << 80 *__p; << 81 }) << 82 << 83 #define raw_cpu_generic_xchg(pcp, nval) << 84 ({ << 85 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)) << 86 typeof(pcp) __ret; << 87 __ret = *__p; << 88 *__p = nval; << 89 __ret; << 90 }) << 91 << 92 #define __cpu_fallback_try_cmpxchg(pcp, ovalp, << 93 ({ << 94 typeof(pcp) __val, __old = *(ovalp); << 95 __val = _cmpxchg(pcp, __old, nval); << 96 if (__val != __old) << 97 *(ovalp) = __val; << 98 __val == __old; << 99 }) << 100 << 101 #define raw_cpu_generic_try_cmpxchg(pcp, ovalp << 102 ({ << 103 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)) << 104 typeof(pcp) __val = *__p, ___old = *(o << 105 bool __ret; << 106 if (__val == ___old) { << 107 *__p = nval; << 108 __ret = true; << 109 } else { << 110 *(ovalp) = __val; << 111 __ret = false; << 112 } << 113 __ret; << 114 }) << 115 << 116 #define raw_cpu_generic_cmpxchg(pcp, oval, nva << 117 ({ << 118 typeof(pcp) __old = (oval); << 119 raw_cpu_generic_try_cmpxchg(pcp, &__ol << 120 __old; << 121 }) << 122 << 123 #define __this_cpu_generic_read_nopreempt(pcp) << 124 ({ << 125 typeof(pcp) ___ret; << 126 preempt_disable_notrace(); << 127 ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp) << 128 preempt_enable_notrace(); << 129 ___ret; << 130 }) << 131 << 132 #define __this_cpu_generic_read_noirq(pcp) << 133 ({ << 134 typeof(pcp) ___ret; << 135 unsigned long ___flags; << 136 raw_local_irq_save(___flags); << 137 ___ret = raw_cpu_generic_read(pcp); << 138 raw_local_irq_restore(___flags); << 139 ___ret; << 140 }) << 141 << 142 #define this_cpu_generic_read(pcp) << 143 ({ << 144 typeof(pcp) __ret; << 145 if (__native_word(pcp)) << 146 __ret = __this_cpu_generic_rea << 147 else << 148 __ret = __this_cpu_generic_rea << 149 __ret; << 150 }) << 151 << 152 #define this_cpu_generic_to_op(pcp, val, op) << 153 do { << 154 unsigned long __flags; << 155 raw_local_irq_save(__flags); << 156 raw_cpu_generic_to_op(pcp, val, op); << 157 raw_local_irq_restore(__flags); << 158 } while (0) << 159 << 160 << 161 #define this_cpu_generic_add_return(pcp, val) << 162 ({ << 163 typeof(pcp) __ret; << 164 unsigned long __flags; << 165 raw_local_irq_save(__flags); << 166 __ret = raw_cpu_generic_add_return(pcp << 167 raw_local_irq_restore(__flags); << 168 __ret; << 169 }) << 170 << 171 #define this_cpu_generic_xchg(pcp, nval) << 172 ({ << 173 typeof(pcp) __ret; << 174 unsigned long __flags; << 175 raw_local_irq_save(__flags); << 176 __ret = raw_cpu_generic_xchg(pcp, nval << 177 raw_local_irq_restore(__flags); << 178 __ret; << 179 }) << 180 << 181 #define this_cpu_generic_try_cmpxchg(pcp, oval << 182 ({ << 183 bool __ret; << 184 unsigned long __flags; << 185 raw_local_irq_save(__flags); << 186 __ret = raw_cpu_generic_try_cmpxchg(pc << 187 raw_local_irq_restore(__flags); << 188 __ret; << 189 }) << 190 << 191 #define this_cpu_generic_cmpxchg(pcp, oval, nv << 192 ({ << 193 typeof(pcp) __ret; << 194 unsigned long __flags; << 195 raw_local_irq_save(__flags); << 196 __ret = raw_cpu_generic_cmpxchg(pcp, o << 197 raw_local_irq_restore(__flags); << 198 __ret; << 199 }) << 200 << 201 #ifndef raw_cpu_read_1 << 202 #define raw_cpu_read_1(pcp) raw_cp << 203 #endif << 204 #ifndef raw_cpu_read_2 << 205 #define raw_cpu_read_2(pcp) raw_cp << 206 #endif << 207 #ifndef raw_cpu_read_4 << 208 #define raw_cpu_read_4(pcp) raw_cp << 209 #endif << 210 #ifndef raw_cpu_read_8 << 211 #define raw_cpu_read_8(pcp) raw_cp << 212 #endif << 213 << 214 #ifndef raw_cpu_write_1 << 215 #define raw_cpu_write_1(pcp, val) raw_cp << 216 #endif << 217 #ifndef raw_cpu_write_2 << 218 #define raw_cpu_write_2(pcp, val) raw_cp << 219 #endif << 220 #ifndef raw_cpu_write_4 << 221 #define raw_cpu_write_4(pcp, val) raw_cp << 222 #endif << 223 #ifndef raw_cpu_write_8 << 224 #define raw_cpu_write_8(pcp, val) raw_cp << 225 #endif << 226 << 227 #ifndef raw_cpu_add_1 << 228 #define raw_cpu_add_1(pcp, val) raw_cp << 229 #endif << 230 #ifndef raw_cpu_add_2 << 231 #define raw_cpu_add_2(pcp, val) raw_cp << 232 #endif << 233 #ifndef raw_cpu_add_4 << 234 #define raw_cpu_add_4(pcp, val) raw_cp << 235 #endif << 236 #ifndef raw_cpu_add_8 << 237 #define raw_cpu_add_8(pcp, val) raw_cp << 238 #endif << 239 << 240 #ifndef raw_cpu_and_1 << 241 #define raw_cpu_and_1(pcp, val) raw_cp << 242 #endif << 243 #ifndef raw_cpu_and_2 << 244 #define raw_cpu_and_2(pcp, val) raw_cp << 245 #endif << 246 #ifndef raw_cpu_and_4 << 247 #define raw_cpu_and_4(pcp, val) raw_cp << 248 #endif << 249 #ifndef raw_cpu_and_8 << 250 #define raw_cpu_and_8(pcp, val) raw_cp << 251 #endif << 252 << 253 #ifndef raw_cpu_or_1 << 254 #define raw_cpu_or_1(pcp, val) raw_cp << 255 #endif << 256 #ifndef raw_cpu_or_2 << 257 #define raw_cpu_or_2(pcp, val) raw_cp << 258 #endif << 259 #ifndef raw_cpu_or_4 << 260 #define raw_cpu_or_4(pcp, val) raw_cp << 261 #endif << 262 #ifndef raw_cpu_or_8 << 263 #define raw_cpu_or_8(pcp, val) raw_cp << 264 #endif << 265 << 266 #ifndef raw_cpu_add_return_1 << 267 #define raw_cpu_add_return_1(pcp, val) raw_cp << 268 #endif << 269 #ifndef raw_cpu_add_return_2 << 270 #define raw_cpu_add_return_2(pcp, val) raw_cp << 271 #endif << 272 #ifndef raw_cpu_add_return_4 << 273 #define raw_cpu_add_return_4(pcp, val) raw_cp << 274 #endif << 275 #ifndef raw_cpu_add_return_8 << 276 #define raw_cpu_add_return_8(pcp, val) raw_cp << 277 #endif << 278 << 279 #ifndef raw_cpu_xchg_1 << 280 #define raw_cpu_xchg_1(pcp, nval) raw_cp << 281 #endif << 282 #ifndef raw_cpu_xchg_2 << 283 #define raw_cpu_xchg_2(pcp, nval) raw_cp << 284 #endif << 285 #ifndef raw_cpu_xchg_4 << 286 #define raw_cpu_xchg_4(pcp, nval) raw_cp << 287 #endif << 288 #ifndef raw_cpu_xchg_8 << 289 #define raw_cpu_xchg_8(pcp, nval) raw_cp << 290 #endif << 291 << 292 #ifndef raw_cpu_try_cmpxchg_1 << 293 #ifdef raw_cpu_cmpxchg_1 << 294 #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval << 295 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 296 #else << 297 #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval << 298 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 299 #endif << 300 #endif << 301 #ifndef raw_cpu_try_cmpxchg_2 << 302 #ifdef raw_cpu_cmpxchg_2 << 303 #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval << 304 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 305 #else << 306 #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval << 307 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 308 #endif << 309 #endif << 310 #ifndef raw_cpu_try_cmpxchg_4 << 311 #ifdef raw_cpu_cmpxchg_4 << 312 #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval << 313 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 314 #else << 315 #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval << 316 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 317 #endif << 318 #endif << 319 #ifndef raw_cpu_try_cmpxchg_8 << 320 #ifdef raw_cpu_cmpxchg_8 << 321 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval << 322 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 323 #else << 324 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval << 325 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 326 #endif << 327 #endif << 328 << 329 #ifndef raw_cpu_try_cmpxchg64 << 330 #ifdef raw_cpu_cmpxchg64 << 331 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval << 332 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 333 #else << 334 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval << 335 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 336 #endif << 337 #endif << 338 #ifndef raw_cpu_try_cmpxchg128 << 339 #ifdef raw_cpu_cmpxchg128 << 340 #define raw_cpu_try_cmpxchg128(pcp, ovalp, nva << 341 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 342 #else << 343 #define raw_cpu_try_cmpxchg128(pcp, ovalp, nva << 344 raw_cpu_generic_try_cmpxchg(pcp, ovalp << 345 #endif << 346 #endif << 347 << 348 #ifndef raw_cpu_cmpxchg_1 << 349 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \ << 350 raw_cpu_generic_cmpxchg(pcp, oval, nva << 351 #endif << 352 #ifndef raw_cpu_cmpxchg_2 << 353 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \ << 354 raw_cpu_generic_cmpxchg(pcp, oval, nva << 355 #endif << 356 #ifndef raw_cpu_cmpxchg_4 << 357 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \ << 358 raw_cpu_generic_cmpxchg(pcp, oval, nva << 359 #endif << 360 #ifndef raw_cpu_cmpxchg_8 << 361 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \ << 362 raw_cpu_generic_cmpxchg(pcp, oval, nva << 363 #endif << 364 << 365 #ifndef raw_cpu_cmpxchg64 << 366 #define raw_cpu_cmpxchg64(pcp, oval, nval) \ << 367 raw_cpu_generic_cmpxchg(pcp, oval, nva << 368 #endif << 369 #ifndef raw_cpu_cmpxchg128 << 370 #define raw_cpu_cmpxchg128(pcp, oval, nval) \ << 371 raw_cpu_generic_cmpxchg(pcp, oval, nva << 372 #endif << 373 << 374 #ifndef this_cpu_read_1 << 375 #define this_cpu_read_1(pcp) this_c << 376 #endif << 377 #ifndef this_cpu_read_2 << 378 #define this_cpu_read_2(pcp) this_c << 379 #endif << 380 #ifndef this_cpu_read_4 << 381 #define this_cpu_read_4(pcp) this_c << 382 #endif << 383 #ifndef this_cpu_read_8 << 384 #define this_cpu_read_8(pcp) this_c << 385 #endif << 386 << 387 #ifndef this_cpu_write_1 << 388 #define this_cpu_write_1(pcp, val) this_c << 389 #endif << 390 #ifndef this_cpu_write_2 << 391 #define this_cpu_write_2(pcp, val) this_c << 392 #endif << 393 #ifndef this_cpu_write_4 << 394 #define this_cpu_write_4(pcp, val) this_c << 395 #endif << 396 #ifndef this_cpu_write_8 << 397 #define this_cpu_write_8(pcp, val) this_c << 398 #endif << 399 << 400 #ifndef this_cpu_add_1 << 401 #define this_cpu_add_1(pcp, val) this_c << 402 #endif << 403 #ifndef this_cpu_add_2 << 404 #define this_cpu_add_2(pcp, val) this_c << 405 #endif << 406 #ifndef this_cpu_add_4 << 407 #define this_cpu_add_4(pcp, val) this_c << 408 #endif << 409 #ifndef this_cpu_add_8 << 410 #define this_cpu_add_8(pcp, val) this_c << 411 #endif << 412 << 413 #ifndef this_cpu_and_1 << 414 #define this_cpu_and_1(pcp, val) this_c << 415 #endif << 416 #ifndef this_cpu_and_2 << 417 #define this_cpu_and_2(pcp, val) this_c << 418 #endif << 419 #ifndef this_cpu_and_4 << 420 #define this_cpu_and_4(pcp, val) this_c << 421 #endif << 422 #ifndef this_cpu_and_8 << 423 #define this_cpu_and_8(pcp, val) this_c << 424 #endif << 425 << 426 #ifndef this_cpu_or_1 << 427 #define this_cpu_or_1(pcp, val) this_c << 428 #endif << 429 #ifndef this_cpu_or_2 << 430 #define this_cpu_or_2(pcp, val) this_c << 431 #endif << 432 #ifndef this_cpu_or_4 << 433 #define this_cpu_or_4(pcp, val) this_c << 434 #endif << 435 #ifndef this_cpu_or_8 << 436 #define this_cpu_or_8(pcp, val) this_c << 437 #endif << 438 << 439 #ifndef this_cpu_add_return_1 << 440 #define this_cpu_add_return_1(pcp, val) this_c << 441 #endif << 442 #ifndef this_cpu_add_return_2 << 443 #define this_cpu_add_return_2(pcp, val) this_c << 444 #endif << 445 #ifndef this_cpu_add_return_4 << 446 #define this_cpu_add_return_4(pcp, val) this_c << 447 #endif << 448 #ifndef this_cpu_add_return_8 << 449 #define this_cpu_add_return_8(pcp, val) this_c << 450 #endif << 451 << 452 #ifndef this_cpu_xchg_1 << 453 #define this_cpu_xchg_1(pcp, nval) this_c << 454 #endif << 455 #ifndef this_cpu_xchg_2 << 456 #define this_cpu_xchg_2(pcp, nval) this_c << 457 #endif << 458 #ifndef this_cpu_xchg_4 << 459 #define this_cpu_xchg_4(pcp, nval) this_c << 460 #endif << 461 #ifndef this_cpu_xchg_8 << 462 #define this_cpu_xchg_8(pcp, nval) this_c << 463 #endif << 464 << 465 #ifndef this_cpu_try_cmpxchg_1 << 466 #ifdef this_cpu_cmpxchg_1 << 467 #define this_cpu_try_cmpxchg_1(pcp, ovalp, nva << 468 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 469 #else << 470 #define this_cpu_try_cmpxchg_1(pcp, ovalp, nva << 471 this_cpu_generic_try_cmpxchg(pcp, oval << 472 #endif << 473 #endif << 474 #ifndef this_cpu_try_cmpxchg_2 << 475 #ifdef this_cpu_cmpxchg_2 << 476 #define this_cpu_try_cmpxchg_2(pcp, ovalp, nva << 477 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 478 #else << 479 #define this_cpu_try_cmpxchg_2(pcp, ovalp, nva << 480 this_cpu_generic_try_cmpxchg(pcp, oval << 481 #endif << 482 #endif << 483 #ifndef this_cpu_try_cmpxchg_4 << 484 #ifdef this_cpu_cmpxchg_4 << 485 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nva << 486 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 487 #else << 488 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nva << 489 this_cpu_generic_try_cmpxchg(pcp, oval << 490 #endif << 491 #endif << 492 #ifndef this_cpu_try_cmpxchg_8 << 493 #ifdef this_cpu_cmpxchg_8 << 494 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nva << 495 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 496 #else << 497 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nva << 498 this_cpu_generic_try_cmpxchg(pcp, oval << 499 #endif << 500 #endif << 501 << 502 #ifndef this_cpu_try_cmpxchg64 << 503 #ifdef this_cpu_cmpxchg64 << 504 #define this_cpu_try_cmpxchg64(pcp, ovalp, nva << 505 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 506 #else << 507 #define this_cpu_try_cmpxchg64(pcp, ovalp, nva << 508 this_cpu_generic_try_cmpxchg(pcp, oval << 509 #endif << 510 #endif << 511 #ifndef this_cpu_try_cmpxchg128 << 512 #ifdef this_cpu_cmpxchg128 << 513 #define this_cpu_try_cmpxchg128(pcp, ovalp, nv << 514 __cpu_fallback_try_cmpxchg(pcp, ovalp, << 515 #else << 516 #define this_cpu_try_cmpxchg128(pcp, ovalp, nv << 517 this_cpu_generic_try_cmpxchg(pcp, oval << 518 #endif << 519 #endif << 520 << 521 #ifndef this_cpu_cmpxchg_1 << 522 #define this_cpu_cmpxchg_1(pcp, oval, nval) \ << 523 this_cpu_generic_cmpxchg(pcp, oval, nv << 524 #endif << 525 #ifndef this_cpu_cmpxchg_2 << 526 #define this_cpu_cmpxchg_2(pcp, oval, nval) \ << 527 this_cpu_generic_cmpxchg(pcp, oval, nv << 528 #endif << 529 #ifndef this_cpu_cmpxchg_4 << 530 #define this_cpu_cmpxchg_4(pcp, oval, nval) \ << 531 this_cpu_generic_cmpxchg(pcp, oval, nv << 532 #endif << 533 #ifndef this_cpu_cmpxchg_8 << 534 #define this_cpu_cmpxchg_8(pcp, oval, nval) \ << 535 this_cpu_generic_cmpxchg(pcp, oval, nv << 536 #endif << 537 << 538 #ifndef this_cpu_cmpxchg64 << 539 #define this_cpu_cmpxchg64(pcp, oval, nval) \ << 540 this_cpu_generic_cmpxchg(pcp, oval, nv << 541 #endif << 542 #ifndef this_cpu_cmpxchg128 << 543 #define this_cpu_cmpxchg128(pcp, oval, nval) \ << 544 this_cpu_generic_cmpxchg(pcp, oval, nv << 545 #endif << 546 << 547 #endif /* _ASM_GENERIC_PERCPU_H_ */ << 548 7
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.