src/WINNT/client_osi/osibasel.c

   1 /*
   2  * Copyright 2000, International Business Machines Corporation and others.
   3  * All Rights Reserved.
   4  *
   5  * This software has been released under the terms of the IBM Public
   6  * License.  For details, see the LICENSE file in the top-level source
   7  * directory or online at http://www.openafs.org/dl/license10.html
   8  */
   9
  10 /* Copyright (C) 1994 Cazamar Systems, Inc. */
  11
  12
  13 #include <afs/param.h>
  14 #include <afs/stds.h>
  15
  16 #include <windows.h>
  17 #include "osi.h"
  18 #include <assert.h>
  19 #include <stdio.h>
  20
  21 /* atomicity-providing critical sections */
  22 CRITICAL_SECTION osi_baseAtomicCS[OSI_MUTEXHASHSIZE];
  23 static long     atomicIndexCounter = 0;
  24
  25 /* Thread local storage index for lock tracking */
  26 static DWORD tls_LockRefH = 0;
  27 static DWORD tls_LockRefT = 0;
  28 static BOOLEAN lockOrderValidation = 0;
  29 static osi_lock_ref_t * lock_ref_FreeListp = NULL;
  30 static osi_lock_ref_t * lock_ref_FreeListEndp = NULL;
  31 CRITICAL_SECTION lock_ref_CS;
  32
  33 void osi_BaseInit(void)
  34 {
  35     int i;
  36
  37     for(i=0; i<OSI_MUTEXHASHSIZE; i++)
  38         InitializeCriticalSectionAndSpinCount(&osi_baseAtomicCS[i], 4000);
  39
  40     if ((tls_LockRefH = TlsAlloc()) == TLS_OUT_OF_INDEXES)
  41         osi_panic("TlsAlloc(tls_LockRefH) failure", __FILE__, __LINE__);
  42
  43     if ((tls_LockRefT = TlsAlloc()) == TLS_OUT_OF_INDEXES)
  44         osi_panic("TlsAlloc(tls_LockRefT) failure", __FILE__, __LINE__);
  45
  46     InitializeCriticalSectionAndSpinCount(&lock_ref_CS, 4000);
  47 }
  48
  49 void
  50 osi_SetLockOrderValidation(int on)
  51 {
  52     lockOrderValidation = (BOOLEAN)on;
  53 }
  54
  55 #ifdef DEBUG
  56 #ifdef _M_IX86
  57 static __inline void
  58 osi_InterlockedAnd(LONG * pdest, LONG value)
  59 {
  60     LONG orig, current, new;
  61
  62     current = *pdest;
  63
  64     do
  65     {
  66         orig = current;
  67         new = orig & value;
  68         current = _InterlockedCompareExchange(pdest, new, orig);
  69     } while (orig != current);
  70 }
  71
  72 static __inline void
  73 osi_InterlockedOr(LONG * pdest, LONG value)
  74 {
  75     LONG orig, current, new;
  76
  77     current = *pdest;
  78
  79     do
  80     {
  81         orig = current;
  82         new = orig | value;
  83         current = _InterlockedCompareExchange(pdest, new, orig);
  84     } while (orig != current);
  85 }
  86
  87 #define _InterlockedOr   osi_InterlockedOr
  88 #define _InterlockedAnd  osi_InterlockedAnd
  89 #endif
  90 #endif
  91
  92 static osi_lock_ref_t *
  93 lock_GetLockRef(void * lockp, char type)
  94 {
  95     osi_lock_ref_t * lockRefp = NULL;
  96
  97     EnterCriticalSection(&lock_ref_CS);
  98     if (lock_ref_FreeListp) {
  99         lockRefp = lock_ref_FreeListp;
 100         osi_QRemoveHT( (osi_queue_t **) &lock_ref_FreeListp,
 101                        (osi_queue_t **) &lock_ref_FreeListEndp,
 102                        &lockRefp->q);
 103     }
 104     LeaveCriticalSection(&lock_ref_CS);
 105
 106     if (lockRefp == NULL)
 107         lockRefp = (osi_lock_ref_t *)malloc(sizeof(osi_lock_ref_t));
 108
 109     memset(lockRefp, 0, sizeof(osi_lock_ref_t));
 110     lockRefp->type = type;
 111     switch (type) {
 112     case OSI_LOCK_MUTEX:
 113         lockRefp->mx = lockp;
 114         break;
 115     case OSI_LOCK_RW:
 116         lockRefp->rw = lockp;
 117         break;
 118     default:
 119         osi_panic("Invalid Lock Type", __FILE__, __LINE__);
 120     }
 121
 122     return lockRefp;
 123 }
 124
 125 static void
 126 lock_FreeLockRef(osi_lock_ref_t * lockRefp)
 127 {
 128     EnterCriticalSection(&lock_ref_CS);
 129     osi_QAddH( (osi_queue_t **) &lock_ref_FreeListp,
 130                (osi_queue_t **) &lock_ref_FreeListEndp,
 131                &lockRefp->q);
 132     LeaveCriticalSection(&lock_ref_CS);
 133 }
 134
 135 void lock_VerifyOrderRW(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_rwlock_t *lockp)
 136 {
 137     char msg[512];
 138     osi_lock_ref_t * lockRefp;
 139
 140     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 141         if (lockRefp->type == OSI_LOCK_RW) {
 142             if (lockRefp->rw == lockp) {
 143                 sprintf(msg, "RW Lock 0x%p level %d already held", lockp, lockp->level);
 144                 osi_panic(msg, __FILE__, __LINE__);
 145             }
 146             if (lockRefp->rw->level > lockp->level) {
 147                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
 148                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
 149                 osi_panic(msg, __FILE__, __LINE__);
 150             }
 151         } else {
 152             if (lockRefp->mx->level > lockp->level) {
 153                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
 154                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
 155                 osi_panic(msg, __FILE__, __LINE__);
 156             }
 157             osi_assertx(lockRefp->mx->level <= lockp->level, "Lock hierarchy violation");
 158         }
 159     }
 160 }
 161
 162 void lock_VerifyOrderMX(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_mutex_t *lockp)
 163 {
 164     char msg[512];
 165     osi_lock_ref_t * lockRefp;
 166
 167     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 168         if (lockRefp->type == OSI_LOCK_MUTEX) {
 169             if (lockRefp->mx == lockp) {
 170                 sprintf(msg, "MX Lock 0x%p level %d already held", lockp, lockp->level);
 171                 osi_panic(msg, __FILE__, __LINE__);
 172             }
 173             if (lockRefp->mx->level > lockp->level) {
 174                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
 175                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
 176                 osi_panic(msg, __FILE__, __LINE__);
 177             }
 178         } else {
 179             if (lockRefp->rw->level > lockp->level) {
 180                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
 181                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
 182                 osi_panic(msg, __FILE__, __LINE__);
 183             }
 184         }
 185     }
 186 }
 187
 188 void lock_ObtainWrite(osi_rwlock_t *lockp)
 189 {
 190     long i;
 191     CRITICAL_SECTION *csp;
 192     osi_queue_t * lockRefH, *lockRefT;
 193     osi_lock_ref_t *lockRefp;
 194     DWORD tid = thrd_Current();
 195
 196     if ((i=lockp->type) != 0) {
 197         if (i >= 0 && i < OSI_NLOCKTYPES)
 198             (osi_lockOps[i]->ObtainWriteProc)(lockp);
 199         return;
 200     }
 201
 202     if (lockOrderValidation) {
 203         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 204         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 205
 206         if (lockp->level != 0)
 207             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
 208     }
 209
 210     /* otherwise we're the fast base type */
 211     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 212     EnterCriticalSection(csp);
 213
 214     if (lockp->flags & OSI_LOCKFLAG_EXCL) {
 215         osi_assertx(lockp->tid[0] != tid, "OSI_RWLOCK_WRITEHELD");
 216     }
 217 #ifdef DEBUG
 218     else {
 219         for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++ ) {
 220             osi_assertx(lockp->tid[i] != tid, "OSI_RWLOCK_READHELD");
 221         }
 222     }
 223 #endif
 224
 225     /* here we have the fast lock, so see if we can obtain the real lock */
 226     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL) ||
 227         (lockp->readers > 0)) {
 228         lockp->waiters++;
 229         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp);
 230         lockp->waiters--;
 231         osi_assertx(lockp->waiters >= 0, "waiters underflow");
 232         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
 233     } else {
 234         /* if we're here, all clear to set the lock */
 235         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
 236         lockp->tid[0] = tid;
 237     }
 238     osi_assertx(lockp->readers == 0, "write lock readers present");
 239
 240     LeaveCriticalSection(csp);
 241
 242     if (lockOrderValidation) {
 243         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
 244         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 245         TlsSetValue(tls_LockRefH, lockRefH);
 246         TlsSetValue(tls_LockRefT, lockRefT);
 247     }
 248 }
 249
 250 void lock_ObtainRead(osi_rwlock_t *lockp)
 251 {
 252     long i;
 253     CRITICAL_SECTION *csp;
 254     osi_queue_t * lockRefH, *lockRefT;
 255     osi_lock_ref_t *lockRefp;
 256     DWORD tid = thrd_Current();
 257
 258     if ((i=lockp->type) != 0) {
 259         if (i >= 0 && i < OSI_NLOCKTYPES)
 260             (osi_lockOps[i]->ObtainReadProc)(lockp);
 261         return;
 262     }
 263
 264     if (lockOrderValidation) {
 265         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 266         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 267
 268         if (lockp->level != 0)
 269             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
 270     }
 271
 272     /* otherwise we're the fast base type */
 273     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 274     EnterCriticalSection(csp);
 275
 276     if (lockp->flags & OSI_LOCKFLAG_EXCL) {
 277         osi_assertx(lockp->tid[0] != tid, "OSI_RWLOCK_WRITEHELD");
 278     }
 279 #ifdef DEBUG
 280     else {
 281         for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++ ) {
 282             osi_assertx(lockp->tid[i] != tid, "OSI_RWLOCK_READHELD");
 283         }
 284     }
 285 #endif
 286
 287     /* here we have the fast lock, so see if we can obtain the real lock */
 288     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
 289         lockp->waiters++;
 290         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4READ, &lockp->readers, lockp->tid, csp);
 291         lockp->waiters--;
 292         osi_assertx(lockp->waiters >= 0, "waiters underflow");
 293         osi_assert(!(lockp->flags & OSI_LOCKFLAG_EXCL) && lockp->readers > 0);
 294     } else {
 295         /* if we're here, all clear to set the lock */
 296         lockp->readers++;
 297 #ifdef DEBUG
 298         if (lockp->readers <= OSI_RWLOCK_THREADS)
 299             lockp->tid[lockp->readers-1] = tid;
 300 #endif
 301     }
 302     LeaveCriticalSection(csp);
 303
 304     if (lockOrderValidation) {
 305         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
 306         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 307         TlsSetValue(tls_LockRefH, lockRefH);
 308         TlsSetValue(tls_LockRefT, lockRefT);
 309     }
 310 }
 311
 312 void lock_ReleaseRead(osi_rwlock_t *lockp)
 313 {
 314     long i;
 315     CRITICAL_SECTION *csp;
 316     osi_queue_t * lockRefH, *lockRefT;
 317     osi_lock_ref_t *lockRefp;
 318     DWORD tid = thrd_Current();
 319
 320     if ((i = lockp->type) != 0) {
 321         if (i >= 0 && i < OSI_NLOCKTYPES)
 322             (osi_lockOps[i]->ReleaseReadProc)(lockp);
 323         return;
 324     }
 325
 326     /* otherwise we're the fast base type */
 327     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 328     EnterCriticalSection(csp);
 329
 330     if (lockOrderValidation && lockp->level != 0) {
 331         int found = 0;
 332         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 333         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 334
 335         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 336             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
 337                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 338                 lock_FreeLockRef(lockRefp);
 339                 found = 1;
 340                 break;
 341             }
 342         }
 343         osi_assertx(found, "read lock not found in TLS queue");
 344
 345         TlsSetValue(tls_LockRefH, lockRefH);
 346         TlsSetValue(tls_LockRefT, lockRefT);
 347     }
 348
 349     osi_assertx(lockp->readers > 0, "read lock not held");
 350
 351 #ifdef DEBUG
 352     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
 353         if ( lockp->tid[i] == tid ) {
 354             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
 355                 lockp->tid[i] = lockp->tid[i+1];
 356             lockp->tid[i] = 0;
 357             break;
 358         }
 359     }
 360 #endif
 361
 362         lockp->readers--;
 363
 364     /* releasing a read lock can allow writers */
 365     if (lockp->readers == 0 && lockp->waiters) {
 366         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
 367     }
 368     else {
 369         osi_assertx(lockp->readers >= 0, "read lock underflow");
 370
 371         /* and finally release the big lock */
 372         LeaveCriticalSection(csp);
 373     }
 374 }
 375
 376 void lock_ReleaseWrite(osi_rwlock_t *lockp)
 377 {
 378     long i;
 379     CRITICAL_SECTION *csp;
 380     osi_queue_t * lockRefH, *lockRefT;
 381     osi_lock_ref_t *lockRefp;
 382
 383     if ((i = lockp->type) != 0) {
 384         if (i >= 0 && i < OSI_NLOCKTYPES)
 385             (osi_lockOps[i]->ReleaseWriteProc)(lockp);
 386         return;
 387     }
 388
 389     /* otherwise we're the fast base type */
 390     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 391     EnterCriticalSection(csp);
 392
 393     if (lockOrderValidation && lockp->level != 0) {
 394         int found = 0;
 395         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 396         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 397
 398         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 399             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
 400                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 401                 lock_FreeLockRef(lockRefp);
 402                 found = 1;
 403                 break;
 404             }
 405         }
 406         osi_assertx(found, "write lock not found in TLS queue");
 407
 408         TlsSetValue(tls_LockRefH, lockRefH);
 409         TlsSetValue(tls_LockRefT, lockRefT);
 410     }
 411
 412     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
 413     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
 414     lockp->tid[0] = 0;
 415
 416     _InterlockedAnd( &lockp->flags, ~OSI_LOCKFLAG_EXCL);
 417     if (lockp->waiters) {
 418         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
 419     }
 420     else {
 421         /* and finally release the big lock */
 422         LeaveCriticalSection(csp);
 423     }
 424 }
 425
 426 void lock_ConvertWToR(osi_rwlock_t *lockp)
 427 {
 428     long i;
 429     CRITICAL_SECTION *csp;
 430
 431     if ((i = lockp->type) != 0) {
 432         if (i >= 0 && i < OSI_NLOCKTYPES)
 433             (osi_lockOps[i]->ConvertWToRProc)(lockp);
 434         return;
 435     }
 436
 437     /* otherwise we're the fast base type */
 438     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 439     EnterCriticalSection(csp);
 440
 441     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
 442     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
 443
 444     /* convert write lock to read lock */
 445     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
 446     lockp->tid[0] = 0;
 447     lockp->readers++;
 448
 449     osi_assertx(lockp->readers == 1, "read lock not one");
 450
 451     if (lockp->waiters) {
 452         osi_TSignalForMLs(&lockp->d.turn, /* still have readers */ 1, csp);
 453     }
 454     else {
 455         /* and finally release the big lock */
 456         LeaveCriticalSection(csp);
 457     }
 458 }
 459
 460 void lock_ConvertRToW(osi_rwlock_t *lockp)
 461 {
 462     long i;
 463     CRITICAL_SECTION *csp;
 464     DWORD tid = thrd_Current();
 465
 466     if ((i = lockp->type) != 0) {
 467         if (i >= 0 && i < OSI_NLOCKTYPES)
 468             (osi_lockOps[i]->ConvertRToWProc)(lockp);
 469         return;
 470     }
 471
 472     /* otherwise we're the fast base type */
 473     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 474     EnterCriticalSection(csp);
 475
 476     osi_assertx(!(lockp->flags & OSI_LOCKFLAG_EXCL), "write lock held");
 477     osi_assertx(lockp->readers > 0, "read lock not held");
 478
 479 #ifdef DEBUG
 480     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
 481         if ( lockp->tid[i] == tid ) {
 482             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
 483                 lockp->tid[i] = lockp->tid[i+1];
 484             lockp->tid[i] = 0;
 485             break;
 486         }
 487     }
 488 #endif
 489
 490     if (--(lockp->readers) == 0) {
 491         /* convert read lock to write lock */
 492         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
 493         lockp->tid[0] = tid;
 494     } else {
 495         osi_assertx(lockp->readers > 0, "read lock underflow");
 496
 497         lockp->waiters++;
 498         osi_TWaitExt(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp, FALSE);
 499         lockp->waiters--;
 500         osi_assertx(lockp->waiters >= 0, "waiters underflow");
 501         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
 502     }
 503
 504     LeaveCriticalSection(csp);
 505 }
 506
 507 void lock_ObtainMutex(struct osi_mutex *lockp)
 508 {
 509     long i;
 510     CRITICAL_SECTION *csp;
 511     osi_queue_t * lockRefH, *lockRefT;
 512     osi_lock_ref_t *lockRefp;
 513
 514     if ((i=lockp->type) != 0) {
 515         if (i >= 0 && i < OSI_NLOCKTYPES)
 516             (osi_lockOps[i]->ObtainMutexProc)(lockp);
 517         return;
 518     }
 519
 520     /* otherwise we're the fast base type */
 521     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 522     EnterCriticalSection(csp);
 523
 524     if (lockOrderValidation) {
 525         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 526         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 527
 528         if (lockp->level != 0)
 529             lock_VerifyOrderMX(lockRefH, lockRefT, lockp);
 530     }
 531
 532     /* here we have the fast lock, so see if we can obtain the real lock */
 533     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
 534         lockp->waiters++;
 535         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, &lockp->tid, csp);
 536         lockp->waiters--;
 537         osi_assertx(lockp->waiters >= 0, "waiters underflow");
 538         osi_assert(lockp->flags & OSI_LOCKFLAG_EXCL);
 539     } else {
 540         /* if we're here, all clear to set the lock */
 541         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
 542         lockp->tid = thrd_Current();
 543     }
 544
 545     LeaveCriticalSection(csp);
 546
 547     if (lockOrderValidation) {
 548         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
 549         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 550         TlsSetValue(tls_LockRefH, lockRefH);
 551         TlsSetValue(tls_LockRefT, lockRefT);
 552     }
 553 }
 554
 555 void lock_ReleaseMutex(struct osi_mutex *lockp)
 556 {
 557     long i;
 558     CRITICAL_SECTION *csp;
 559     osi_queue_t * lockRefH, *lockRefT;
 560     osi_lock_ref_t *lockRefp;
 561
 562     if ((i = lockp->type) != 0) {
 563         if (i >= 0 && i < OSI_NLOCKTYPES)
 564             (osi_lockOps[i]->ReleaseMutexProc)(lockp);
 565         return;
 566     }
 567
 568     /* otherwise we're the fast base type */
 569     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 570     EnterCriticalSection(csp);
 571
 572     if (lockOrderValidation && lockp->level != 0) {
 573         int found = 0;
 574         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 575         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 576
 577         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 578             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
 579                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 580                 lock_FreeLockRef(lockRefp);
 581                 found = 1;
 582                 break;
 583             }
 584         }
 585
 586         osi_assertx(found, "mutex lock not found in TLS queue");
 587         TlsSetValue(tls_LockRefH, lockRefH);
 588         TlsSetValue(tls_LockRefT, lockRefT);
 589     }
 590
 591     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "mutex not held");
 592     osi_assertx(lockp->tid == thrd_Current(), "mutex not held by current thread");
 593
 594     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
 595     lockp->tid = 0;
 596     if (lockp->waiters) {
 597         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
 598     }
 599     else {
 600         /* and finally release the big lock */
 601         LeaveCriticalSection(csp);
 602     }
 603 }
 604
 605 int lock_TryRead(struct osi_rwlock *lockp)
 606 {
 607     long i;
 608     CRITICAL_SECTION *csp;
 609     osi_queue_t * lockRefH, *lockRefT;
 610     osi_lock_ref_t *lockRefp;
 611
 612     if ((i=lockp->type) != 0)
 613         if (i >= 0 && i < OSI_NLOCKTYPES)
 614             return (osi_lockOps[i]->TryReadProc)(lockp);
 615
 616     /* otherwise we're the fast base type */
 617     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 618     EnterCriticalSection(csp);
 619
 620     if (lockOrderValidation) {
 621         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 622         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 623
 624         if (lockp->level != 0) {
 625             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 626                 if (lockRefp->type == OSI_LOCK_RW) {
 627                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
 628                 }
 629             }
 630         }
 631     }
 632
 633     /* here we have the fast lock, so see if we can obtain the real lock */
 634     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
 635         i = 0;
 636     }
 637     else {
 638         /* if we're here, all clear to set the lock */
 639         lockp->readers++;
 640 #ifdef DEBUG
 641         if (lockp->readers < OSI_RWLOCK_THREADS)
 642             lockp->tid[lockp->readers-1] = thrd_Current();
 643 #endif
 644         i = 1;
 645     }
 646
 647     LeaveCriticalSection(csp);
 648
 649     if (lockOrderValidation && i) {
 650         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
 651         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 652         TlsSetValue(tls_LockRefH, lockRefH);
 653         TlsSetValue(tls_LockRefT, lockRefT);
 654     }
 655
 656     return i;
 657 }
 658
 659
 660 int lock_TryWrite(struct osi_rwlock *lockp)
 661 {
 662     long i;
 663     CRITICAL_SECTION *csp;
 664     osi_queue_t * lockRefH, *lockRefT;
 665     osi_lock_ref_t *lockRefp;
 666
 667     if ((i=lockp->type) != 0)
 668         if (i >= 0 && i < OSI_NLOCKTYPES)
 669             return (osi_lockOps[i]->TryWriteProc)(lockp);
 670
 671     /* otherwise we're the fast base type */
 672     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 673     EnterCriticalSection(csp);
 674
 675     if (lockOrderValidation) {
 676         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 677         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 678
 679         if (lockp->level != 0) {
 680             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 681                 if (lockRefp->type == OSI_LOCK_RW) {
 682                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
 683                 }
 684             }
 685         }
 686     }
 687
 688     /* here we have the fast lock, so see if we can obtain the real lock */
 689     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)
 690          || (lockp->readers > 0)) {
 691         i = 0;
 692     }
 693     else {
 694         /* if we're here, all clear to set the lock */
 695         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
 696         lockp->tid[0] = thrd_Current();
 697         i = 1;
 698     }
 699
 700     LeaveCriticalSection(csp);
 701
 702     if (lockOrderValidation && i) {
 703         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
 704         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 705         TlsSetValue(tls_LockRefH, lockRefH);
 706         TlsSetValue(tls_LockRefT, lockRefT);
 707     }
 708
 709     return i;
 710 }
 711
 712
 713 int lock_TryMutex(struct osi_mutex *lockp) {
 714     long i;
 715     CRITICAL_SECTION *csp;
 716     osi_queue_t * lockRefH, *lockRefT;
 717     osi_lock_ref_t *lockRefp;
 718
 719     if ((i=lockp->type) != 0)
 720         if (i >= 0 && i < OSI_NLOCKTYPES)
 721             return (osi_lockOps[i]->TryMutexProc)(lockp);
 722
 723     /* otherwise we're the fast base type */
 724     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 725     EnterCriticalSection(csp);
 726
 727     if (lockOrderValidation) {
 728         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 729         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 730
 731         if (lockp->level != 0) {
 732             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 733                 if (lockRefp->type == OSI_LOCK_MUTEX) {
 734                     osi_assertx(lockRefp->mx != lockp, "Mutex already held");
 735                 }
 736             }
 737         }
 738     }
 739
 740     /* here we have the fast lock, so see if we can obtain the real lock */
 741     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
 742         i = 0;
 743     }
 744     else {
 745         /* if we're here, all clear to set the lock */
 746         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
 747         lockp->tid = thrd_Current();
 748         i = 1;
 749     }
 750
 751     LeaveCriticalSection(csp);
 752
 753     if (lockOrderValidation && i) {
 754         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
 755         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
 756         TlsSetValue(tls_LockRefH, lockRefH);
 757         TlsSetValue(tls_LockRefT, lockRefT);
 758     }
 759
 760     return i;
 761 }
 762
 763 void osi_SleepR(LONG_PTR sleepVal, struct osi_rwlock *lockp)
 764 {
 765     long i;
 766     CRITICAL_SECTION *csp;
 767     osi_queue_t * lockRefH, *lockRefT;
 768     osi_lock_ref_t *lockRefp;
 769     DWORD tid = thrd_Current();
 770
 771     if ((i = lockp->type) != 0) {
 772         if (i >= 0 && i < OSI_NLOCKTYPES)
 773             (osi_lockOps[i]->SleepRProc)(sleepVal, lockp);
 774         return;
 775     }
 776
 777     /* otherwise we're the fast base type */
 778     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 779     EnterCriticalSection(csp);
 780
 781     if (lockOrderValidation && lockp->level != 0) {
 782         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 783         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 784
 785         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 786             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
 787                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 788                 lock_FreeLockRef(lockRefp);
 789                 break;
 790             }
 791         }
 792
 793         TlsSetValue(tls_LockRefH, lockRefH);
 794         TlsSetValue(tls_LockRefT, lockRefT);
 795     }
 796
 797     osi_assertx(lockp->readers > 0, "osi_SleepR: not held");
 798
 799 #ifdef DEBUG
 800     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
 801         if ( lockp->tid[i] == tid ) {
 802             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
 803                 lockp->tid[i] = lockp->tid[i+1];
 804             lockp->tid[i] = 0;
 805             break;
 806         }
 807     }
 808 #endif
 809
 810     /* XXX better to get the list of things to wakeup from TSignalForMLs, and
 811      * then do the wakeup after SleepSpin releases the low-level mutex.
 812      */
 813     if (--(lockp->readers) == 0 && lockp->waiters) {
 814         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
 815     }
 816
 817     /* now call into scheduler to sleep atomically with releasing spin lock */
 818     osi_SleepSpin(sleepVal, csp);
 819 }
 820
 821 void osi_SleepW(LONG_PTR sleepVal, struct osi_rwlock *lockp)
 822 {
 823     long i;
 824     CRITICAL_SECTION *csp;
 825     osi_queue_t * lockRefH, *lockRefT;
 826     osi_lock_ref_t *lockRefp;
 827     DWORD tid = thrd_Current();
 828
 829     if ((i = lockp->type) != 0) {
 830         if (i >= 0 && i < OSI_NLOCKTYPES)
 831             (osi_lockOps[i]->SleepWProc)(sleepVal, lockp);
 832         return;
 833     }
 834
 835     /* otherwise we're the fast base type */
 836     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 837     EnterCriticalSection(csp);
 838
 839     if (lockOrderValidation && lockp->level != 0) {
 840         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 841         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 842
 843         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 844             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
 845                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 846                 lock_FreeLockRef(lockRefp);
 847                 break;
 848             }
 849         }
 850
 851         TlsSetValue(tls_LockRefH, lockRefH);
 852         TlsSetValue(tls_LockRefT, lockRefT);
 853     }
 854
 855     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepW: not held");
 856
 857     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
 858     lockp->tid[0] = 0;
 859     if (lockp->waiters) {
 860         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
 861     }
 862
 863     /* and finally release the big lock */
 864     osi_SleepSpin(sleepVal, csp);
 865 }
 866
 867 void osi_SleepM(LONG_PTR sleepVal, struct osi_mutex *lockp)
 868 {
 869     long i;
 870     CRITICAL_SECTION *csp;
 871     osi_queue_t * lockRefH, *lockRefT;
 872     osi_lock_ref_t *lockRefp;
 873
 874     if ((i = lockp->type) != 0) {
 875         if (i >= 0 && i < OSI_NLOCKTYPES)
 876             (osi_lockOps[i]->SleepMProc)(sleepVal, lockp);
 877         return;
 878     }
 879
 880     /* otherwise we're the fast base type */
 881     csp = &osi_baseAtomicCS[lockp->atomicIndex];
 882     EnterCriticalSection(csp);
 883
 884     if (lockOrderValidation && lockp->level != 0) {
 885         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
 886         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
 887
 888         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
 889             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
 890                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
 891                 lock_FreeLockRef(lockRefp);
 892                 break;
 893             }
 894         }
 895
 896         TlsSetValue(tls_LockRefH, lockRefH);
 897         TlsSetValue(tls_LockRefT, lockRefT);
 898     }
 899
 900     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepM not held");
 901
 902     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
 903     lockp->tid = 0;
 904     if (lockp->waiters) {
 905         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
 906     }
 907
 908     /* and finally release the big lock */
 909     osi_SleepSpin(sleepVal, csp);
 910 }
 911
 912 void lock_FinalizeRWLock(osi_rwlock_t *lockp)
 913 {
 914     long i;
 915
 916     if ((i=lockp->type) != 0)
 917         if (i >= 0 && i < OSI_NLOCKTYPES)
 918             (osi_lockOps[i]->FinalizeRWLockProc)(lockp);
 919 }
 920
 921 void lock_FinalizeMutex(osi_mutex_t *lockp)
 922 {
 923     long i;
 924
 925     if ((i=lockp->type) != 0)
 926         if (i >= 0 && i < OSI_NLOCKTYPES)
 927             (osi_lockOps[i]->FinalizeMutexProc)(lockp);
 928 }
 929
 930 void lock_InitializeMutex(osi_mutex_t *mp, char *namep, unsigned short level)
 931 {
 932     int i;
 933
 934     if ((i = osi_lockTypeDefault) > 0) {
 935         if (i >= 0 && i < OSI_NLOCKTYPES)
 936             (osi_lockOps[i]->InitializeMutexProc)(mp, namep, level);
 937         return;
 938     }
 939
 940     /*
 941      * otherwise we have the base case, which requires no special
 942      * initialization.
 943      */
 944     memset(mp, 0, sizeof(osi_mutex_t));
 945     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
 946     mp->level = level;
 947     osi_TInit(&mp->d.turn);
 948     return;
 949 }
 950
 951 void lock_InitializeRWLock(osi_rwlock_t *mp, char *namep, unsigned short level)
 952 {
 953     int i;
 954
 955     if ((i = osi_lockTypeDefault) > 0) {
 956         if (i >= 0 && i < OSI_NLOCKTYPES)
 957             (osi_lockOps[i]->InitializeRWLockProc)(mp, namep, level);
 958         return;
 959     }
 960
 961     /* otherwise we have the base case, which requires no special
 962      * initialization.
 963      */
 964     memset(mp, 0, sizeof(osi_rwlock_t));
 965     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
 966     mp->level = level;
 967     osi_TInit(&mp->d.turn);
 968     return;
 969 }
 970
 971 int lock_GetRWLockState(osi_rwlock_t *lp)
 972 {
 973     long i;
 974     CRITICAL_SECTION *csp;
 975
 976     if ((i=lp->type) != 0)
 977         if (i >= 0 && i < OSI_NLOCKTYPES)
 978             return (osi_lockOps[i]->GetRWLockState)(lp);
 979
 980     /* otherwise we're the fast base type */
 981     csp = &osi_baseAtomicCS[lp->atomicIndex];
 982     EnterCriticalSection(csp);
 983
 984     /* here we have the fast lock, so see if we can obtain the real lock */
 985     if (lp->flags & OSI_LOCKFLAG_EXCL)
 986         i = OSI_RWLOCK_WRITEHELD;
 987     else
 988         i = 0;
 989     if (lp->readers > 0)
 990         i |= OSI_RWLOCK_READHELD;
 991
 992     LeaveCriticalSection(csp);
 993
 994     return i;
 995 }
 996
 997 int lock_GetMutexState(struct osi_mutex *mp)
 998 {
 999     long i;
1000     CRITICAL_SECTION *csp;
1001
1002     if ((i=mp->type) != 0)
1003         if (i >= 0 && i < OSI_NLOCKTYPES)
1004             return (osi_lockOps[i]->GetMutexState)(mp);
1005
1006     /* otherwise we're the fast base type */
1007     csp = &osi_baseAtomicCS[mp->atomicIndex];
1008     EnterCriticalSection(csp);
1009
1010     if (mp->flags & OSI_LOCKFLAG_EXCL)
1011         i = OSI_MUTEX_HELD;
1012     else
1013         i = 0;
1014
1015     LeaveCriticalSection(csp);
1016
1017     return i;
1018 }