Windows: interlocked ops for osi lock flags
[openafs.git] / src / WINNT / client_osi / osibasel.c
1 /*
2  * Copyright 2000, International Business Machines Corporation and others.
3  * All Rights Reserved.
4  *
5  * This software has been released under the terms of the IBM Public
6  * License.  For details, see the LICENSE file in the top-level source
7  * directory or online at http://www.openafs.org/dl/license10.html
8  */
9
10 /* Copyright (C) 1994 Cazamar Systems, Inc. */
11
12
13 #include <afs/param.h>
14 #include <afs/stds.h>
15
16 #include <windows.h>
17 #include "osi.h"
18 #include <assert.h>
19 #include <stdio.h>
20
21 /* atomicity-providing critical sections */
22 CRITICAL_SECTION osi_baseAtomicCS[OSI_MUTEXHASHSIZE];
23 static long     atomicIndexCounter = 0;
24
25 /* Thread local storage index for lock tracking */
26 static DWORD tls_LockRefH = 0;
27 static DWORD tls_LockRefT = 0;
28 static BOOLEAN lockOrderValidation = 0;
29 static osi_lock_ref_t * lock_ref_FreeListp = NULL;
30 static osi_lock_ref_t * lock_ref_FreeListEndp = NULL;
31 CRITICAL_SECTION lock_ref_CS;
32
33 void osi_BaseInit(void)
34 {
35     int i;
36
37     for(i=0; i<OSI_MUTEXHASHSIZE; i++)
38         InitializeCriticalSectionAndSpinCount(&osi_baseAtomicCS[i], 4000);
39
40     if ((tls_LockRefH = TlsAlloc()) == TLS_OUT_OF_INDEXES)
41         osi_panic("TlsAlloc(tls_LockRefH) failure", __FILE__, __LINE__);
42
43     if ((tls_LockRefT = TlsAlloc()) == TLS_OUT_OF_INDEXES)
44         osi_panic("TlsAlloc(tls_LockRefT) failure", __FILE__, __LINE__);
45
46     InitializeCriticalSectionAndSpinCount(&lock_ref_CS, 4000);
47 }
48
49 void
50 osi_SetLockOrderValidation(int on)
51 {
52     lockOrderValidation = (BOOLEAN)on;
53 }
54
55 #ifdef DEBUG
56 #ifdef _M_IX86
57 static __inline void
58 osi_InterlockedAnd(LONG * pdest, LONG value)
59 {
60     LONG orig, current, new;
61
62     current = *pdest;
63
64     do
65     {
66         orig = current;
67         new = orig & value;
68         current = _InterlockedCompareExchange(pdest, new, orig);
69     } while (orig != current);
70 }
71
72 static __inline void
73 osi_InterlockedOr(LONG * pdest, LONG value)
74 {
75     LONG orig, current, new;
76
77     current = *pdest;
78
79     do
80     {
81         orig = current;
82         new = orig | value;
83         current = _InterlockedCompareExchange(pdest, new, orig);
84     } while (orig != current);
85 }
86
87 #define _InterlockedOr   osi_InterlockedOr
88 #define _InterlockedAnd  osi_InterlockedAnd
89 #endif
90 #endif
91
92 static osi_lock_ref_t *
93 lock_GetLockRef(void * lockp, char type)
94 {
95     osi_lock_ref_t * lockRefp = NULL;
96
97     EnterCriticalSection(&lock_ref_CS);
98     if (lock_ref_FreeListp) {
99         lockRefp = lock_ref_FreeListp;
100         osi_QRemoveHT( (osi_queue_t **) &lock_ref_FreeListp,
101                        (osi_queue_t **) &lock_ref_FreeListEndp,
102                        &lockRefp->q);
103     }
104     LeaveCriticalSection(&lock_ref_CS);
105
106     if (lockRefp == NULL)
107         lockRefp = (osi_lock_ref_t *)malloc(sizeof(osi_lock_ref_t));
108
109     memset(lockRefp, 0, sizeof(osi_lock_ref_t));
110     lockRefp->type = type;
111     switch (type) {
112     case OSI_LOCK_MUTEX:
113         lockRefp->mx = lockp;
114         break;
115     case OSI_LOCK_RW:
116         lockRefp->rw = lockp;
117         break;
118     default:
119         osi_panic("Invalid Lock Type", __FILE__, __LINE__);
120     }
121
122     return lockRefp;
123 }
124
125 static void
126 lock_FreeLockRef(osi_lock_ref_t * lockRefp)
127 {
128     EnterCriticalSection(&lock_ref_CS);
129     osi_QAddH( (osi_queue_t **) &lock_ref_FreeListp,
130                (osi_queue_t **) &lock_ref_FreeListEndp,
131                &lockRefp->q);
132     LeaveCriticalSection(&lock_ref_CS);
133 }
134
135 void lock_VerifyOrderRW(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_rwlock_t *lockp)
136 {
137     char msg[512];
138     osi_lock_ref_t * lockRefp;
139
140     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
141         if (lockRefp->type == OSI_LOCK_RW) {
142             if (lockRefp->rw == lockp) {
143                 sprintf(msg, "RW Lock 0x%p level %d already held", lockp, lockp->level);
144                 osi_panic(msg, __FILE__, __LINE__);
145             }
146             if (lockRefp->rw->level > lockp->level) {
147                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
148                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
149                 osi_panic(msg, __FILE__, __LINE__);
150             }
151         } else {
152             if (lockRefp->mx->level > lockp->level) {
153                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
154                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
155                 osi_panic(msg, __FILE__, __LINE__);
156             }
157             osi_assertx(lockRefp->mx->level <= lockp->level, "Lock hierarchy violation");
158         }
159     }
160 }
161
162 void lock_VerifyOrderMX(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_mutex_t *lockp)
163 {
164     char msg[512];
165     osi_lock_ref_t * lockRefp;
166
167     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
168         if (lockRefp->type == OSI_LOCK_MUTEX) {
169             if (lockRefp->mx == lockp) {
170                 sprintf(msg, "MX Lock 0x%p level %d already held", lockp, lockp->level);
171                 osi_panic(msg, __FILE__, __LINE__);
172             }
173             if (lockRefp->mx->level > lockp->level) {
174                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
175                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
176                 osi_panic(msg, __FILE__, __LINE__);
177             }
178         } else {
179             if (lockRefp->rw->level > lockp->level) {
180                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
181                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
182                 osi_panic(msg, __FILE__, __LINE__);
183             }
184         }
185     }
186 }
187
188 void lock_ObtainWrite(osi_rwlock_t *lockp)
189 {
190     long i;
191     CRITICAL_SECTION *csp;
192     osi_queue_t * lockRefH, *lockRefT;
193     osi_lock_ref_t *lockRefp;
194     DWORD tid = thrd_Current();
195
196     if ((i=lockp->type) != 0) {
197         if (i >= 0 && i < OSI_NLOCKTYPES)
198             (osi_lockOps[i]->ObtainWriteProc)(lockp);
199         return;
200     }
201
202     if (lockOrderValidation) {
203         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
204         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
205
206         if (lockp->level != 0)
207             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
208     }
209
210     /* otherwise we're the fast base type */
211     csp = &osi_baseAtomicCS[lockp->atomicIndex];
212     EnterCriticalSection(csp);
213
214     if (lockp->flags & OSI_LOCKFLAG_EXCL) {
215         osi_assertx(lockp->tid[0] != tid, "OSI_RWLOCK_WRITEHELD");
216     }
217 #ifdef DEBUG
218     else {
219         for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++ ) {
220             osi_assertx(lockp->tid[i] != tid, "OSI_RWLOCK_READHELD");
221         }
222     }
223 #endif
224
225     /* here we have the fast lock, so see if we can obtain the real lock */
226     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL) ||
227         (lockp->readers > 0)) {
228         lockp->waiters++;
229         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp);
230         lockp->waiters--;
231         osi_assertx(lockp->waiters >= 0, "waiters underflow");
232         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
233     } else {
234         /* if we're here, all clear to set the lock */
235         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
236         lockp->tid[0] = tid;
237     }
238     osi_assertx(lockp->readers == 0, "write lock readers present");
239
240     LeaveCriticalSection(csp);
241
242     if (lockOrderValidation) {
243         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
244         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
245         TlsSetValue(tls_LockRefH, lockRefH);
246         TlsSetValue(tls_LockRefT, lockRefT);
247     }
248 }
249
250 void lock_ObtainRead(osi_rwlock_t *lockp)
251 {
252     long i;
253     CRITICAL_SECTION *csp;
254     osi_queue_t * lockRefH, *lockRefT;
255     osi_lock_ref_t *lockRefp;
256     DWORD tid = thrd_Current();
257
258     if ((i=lockp->type) != 0) {
259         if (i >= 0 && i < OSI_NLOCKTYPES)
260             (osi_lockOps[i]->ObtainReadProc)(lockp);
261         return;
262     }
263
264     if (lockOrderValidation) {
265         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
266         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
267
268         if (lockp->level != 0)
269             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
270     }
271
272     /* otherwise we're the fast base type */
273     csp = &osi_baseAtomicCS[lockp->atomicIndex];
274     EnterCriticalSection(csp);
275
276     if (lockp->flags & OSI_LOCKFLAG_EXCL) {
277         osi_assertx(lockp->tid[0] != tid, "OSI_RWLOCK_WRITEHELD");
278     }
279 #ifdef DEBUG
280     else {
281         for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++ ) {
282             osi_assertx(lockp->tid[i] != tid, "OSI_RWLOCK_READHELD");
283         }
284     }
285 #endif
286
287     /* here we have the fast lock, so see if we can obtain the real lock */
288     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
289         lockp->waiters++;
290         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4READ, &lockp->readers, lockp->tid, csp);
291         lockp->waiters--;
292         osi_assertx(lockp->waiters >= 0, "waiters underflow");
293         osi_assert(!(lockp->flags & OSI_LOCKFLAG_EXCL) && lockp->readers > 0);
294     } else {
295         /* if we're here, all clear to set the lock */
296         lockp->readers++;
297 #ifdef DEBUG
298         if (lockp->readers <= OSI_RWLOCK_THREADS)
299             lockp->tid[lockp->readers-1] = tid;
300 #endif
301     }
302     LeaveCriticalSection(csp);
303
304     if (lockOrderValidation) {
305         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
306         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
307         TlsSetValue(tls_LockRefH, lockRefH);
308         TlsSetValue(tls_LockRefT, lockRefT);
309     }
310 }
311
312 void lock_ReleaseRead(osi_rwlock_t *lockp)
313 {
314     long i;
315     CRITICAL_SECTION *csp;
316     osi_queue_t * lockRefH, *lockRefT;
317     osi_lock_ref_t *lockRefp;
318     DWORD tid = thrd_Current();
319
320     if ((i = lockp->type) != 0) {
321         if (i >= 0 && i < OSI_NLOCKTYPES)
322             (osi_lockOps[i]->ReleaseReadProc)(lockp);
323         return;
324     }
325
326     /* otherwise we're the fast base type */
327     csp = &osi_baseAtomicCS[lockp->atomicIndex];
328     EnterCriticalSection(csp);
329
330     if (lockOrderValidation && lockp->level != 0) {
331         int found = 0;
332         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
333         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
334
335         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
336             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
337                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
338                 lock_FreeLockRef(lockRefp);
339                 found = 1;
340                 break;
341             }
342         }
343         osi_assertx(found, "read lock not found in TLS queue");
344
345         TlsSetValue(tls_LockRefH, lockRefH);
346         TlsSetValue(tls_LockRefT, lockRefT);
347     }
348
349     osi_assertx(lockp->readers > 0, "read lock not held");
350
351 #ifdef DEBUG
352     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
353         if ( lockp->tid[i] == tid ) {
354             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
355                 lockp->tid[i] = lockp->tid[i+1];
356             lockp->tid[i] = 0;
357             break;
358         }
359     }
360 #endif
361
362         lockp->readers--;
363
364     /* releasing a read lock can allow writers */
365     if (lockp->readers == 0 && lockp->waiters) {
366         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
367     }
368     else {
369         osi_assertx(lockp->readers >= 0, "read lock underflow");
370
371         /* and finally release the big lock */
372         LeaveCriticalSection(csp);
373     }
374 }
375
376 void lock_ReleaseWrite(osi_rwlock_t *lockp)
377 {
378     long i;
379     CRITICAL_SECTION *csp;
380     osi_queue_t * lockRefH, *lockRefT;
381     osi_lock_ref_t *lockRefp;
382
383     if ((i = lockp->type) != 0) {
384         if (i >= 0 && i < OSI_NLOCKTYPES)
385             (osi_lockOps[i]->ReleaseWriteProc)(lockp);
386         return;
387     }
388
389     /* otherwise we're the fast base type */
390     csp = &osi_baseAtomicCS[lockp->atomicIndex];
391     EnterCriticalSection(csp);
392
393     if (lockOrderValidation && lockp->level != 0) {
394         int found = 0;
395         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
396         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
397
398         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
399             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
400                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
401                 lock_FreeLockRef(lockRefp);
402                 found = 1;
403                 break;
404             }
405         }
406         osi_assertx(found, "write lock not found in TLS queue");
407
408         TlsSetValue(tls_LockRefH, lockRefH);
409         TlsSetValue(tls_LockRefT, lockRefT);
410     }
411
412     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
413     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
414     lockp->tid[0] = 0;
415
416     _InterlockedAnd( &lockp->flags, ~OSI_LOCKFLAG_EXCL);
417     if (lockp->waiters) {
418         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
419     }
420     else {
421         /* and finally release the big lock */
422         LeaveCriticalSection(csp);
423     }
424 }
425
426 void lock_ConvertWToR(osi_rwlock_t *lockp)
427 {
428     long i;
429     CRITICAL_SECTION *csp;
430
431     if ((i = lockp->type) != 0) {
432         if (i >= 0 && i < OSI_NLOCKTYPES)
433             (osi_lockOps[i]->ConvertWToRProc)(lockp);
434         return;
435     }
436
437     /* otherwise we're the fast base type */
438     csp = &osi_baseAtomicCS[lockp->atomicIndex];
439     EnterCriticalSection(csp);
440
441     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
442     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
443
444     /* convert write lock to read lock */
445     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
446     lockp->tid[0] = 0;
447     lockp->readers++;
448
449     osi_assertx(lockp->readers == 1, "read lock not one");
450
451     if (lockp->waiters) {
452         osi_TSignalForMLs(&lockp->d.turn, /* still have readers */ 1, csp);
453     }
454     else {
455         /* and finally release the big lock */
456         LeaveCriticalSection(csp);
457     }
458 }
459
460 void lock_ConvertRToW(osi_rwlock_t *lockp)
461 {
462     long i;
463     CRITICAL_SECTION *csp;
464     DWORD tid = thrd_Current();
465
466     if ((i = lockp->type) != 0) {
467         if (i >= 0 && i < OSI_NLOCKTYPES)
468             (osi_lockOps[i]->ConvertRToWProc)(lockp);
469         return;
470     }
471
472     /* otherwise we're the fast base type */
473     csp = &osi_baseAtomicCS[lockp->atomicIndex];
474     EnterCriticalSection(csp);
475
476     osi_assertx(!(lockp->flags & OSI_LOCKFLAG_EXCL), "write lock held");
477     osi_assertx(lockp->readers > 0, "read lock not held");
478
479 #ifdef DEBUG
480     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
481         if ( lockp->tid[i] == tid ) {
482             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
483                 lockp->tid[i] = lockp->tid[i+1];
484             lockp->tid[i] = 0;
485             break;
486         }
487     }
488 #endif
489
490     if (--(lockp->readers) == 0) {
491         /* convert read lock to write lock */
492         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
493         lockp->tid[0] = tid;
494     } else {
495         osi_assertx(lockp->readers > 0, "read lock underflow");
496
497         lockp->waiters++;
498         osi_TWaitExt(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp, FALSE);
499         lockp->waiters--;
500         osi_assertx(lockp->waiters >= 0, "waiters underflow");
501         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
502     }
503
504     LeaveCriticalSection(csp);
505 }
506
507 void lock_ObtainMutex(struct osi_mutex *lockp)
508 {
509     long i;
510     CRITICAL_SECTION *csp;
511     osi_queue_t * lockRefH, *lockRefT;
512     osi_lock_ref_t *lockRefp;
513
514     if ((i=lockp->type) != 0) {
515         if (i >= 0 && i < OSI_NLOCKTYPES)
516             (osi_lockOps[i]->ObtainMutexProc)(lockp);
517         return;
518     }
519
520     /* otherwise we're the fast base type */
521     csp = &osi_baseAtomicCS[lockp->atomicIndex];
522     EnterCriticalSection(csp);
523
524     if (lockOrderValidation) {
525         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
526         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
527
528         if (lockp->level != 0)
529             lock_VerifyOrderMX(lockRefH, lockRefT, lockp);
530     }
531
532     /* here we have the fast lock, so see if we can obtain the real lock */
533     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
534         lockp->waiters++;
535         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, &lockp->tid, csp);
536         lockp->waiters--;
537         osi_assertx(lockp->waiters >= 0, "waiters underflow");
538         osi_assert(lockp->flags & OSI_LOCKFLAG_EXCL);
539     } else {
540         /* if we're here, all clear to set the lock */
541         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
542         lockp->tid = thrd_Current();
543     }
544
545     LeaveCriticalSection(csp);
546
547     if (lockOrderValidation) {
548         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
549         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
550         TlsSetValue(tls_LockRefH, lockRefH);
551         TlsSetValue(tls_LockRefT, lockRefT);
552     }
553 }
554
555 void lock_ReleaseMutex(struct osi_mutex *lockp)
556 {
557     long i;
558     CRITICAL_SECTION *csp;
559     osi_queue_t * lockRefH, *lockRefT;
560     osi_lock_ref_t *lockRefp;
561
562     if ((i = lockp->type) != 0) {
563         if (i >= 0 && i < OSI_NLOCKTYPES)
564             (osi_lockOps[i]->ReleaseMutexProc)(lockp);
565         return;
566     }
567
568     /* otherwise we're the fast base type */
569     csp = &osi_baseAtomicCS[lockp->atomicIndex];
570     EnterCriticalSection(csp);
571
572     if (lockOrderValidation && lockp->level != 0) {
573         int found = 0;
574         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
575         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
576
577         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
578             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
579                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
580                 lock_FreeLockRef(lockRefp);
581                 found = 1;
582                 break;
583             }
584         }
585
586         osi_assertx(found, "mutex lock not found in TLS queue");
587         TlsSetValue(tls_LockRefH, lockRefH);
588         TlsSetValue(tls_LockRefT, lockRefT);
589     }
590
591     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "mutex not held");
592     osi_assertx(lockp->tid == thrd_Current(), "mutex not held by current thread");
593
594     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
595     lockp->tid = 0;
596     if (lockp->waiters) {
597         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
598     }
599     else {
600         /* and finally release the big lock */
601         LeaveCriticalSection(csp);
602     }
603 }
604
605 int lock_TryRead(struct osi_rwlock *lockp)
606 {
607     long i;
608     CRITICAL_SECTION *csp;
609     osi_queue_t * lockRefH, *lockRefT;
610     osi_lock_ref_t *lockRefp;
611
612     if ((i=lockp->type) != 0)
613         if (i >= 0 && i < OSI_NLOCKTYPES)
614             return (osi_lockOps[i]->TryReadProc)(lockp);
615
616     /* otherwise we're the fast base type */
617     csp = &osi_baseAtomicCS[lockp->atomicIndex];
618     EnterCriticalSection(csp);
619
620     if (lockOrderValidation) {
621         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
622         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
623
624         if (lockp->level != 0) {
625             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
626                 if (lockRefp->type == OSI_LOCK_RW) {
627                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
628                 }
629             }
630         }
631     }
632
633     /* here we have the fast lock, so see if we can obtain the real lock */
634     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
635         i = 0;
636     }
637     else {
638         /* if we're here, all clear to set the lock */
639         lockp->readers++;
640 #ifdef DEBUG
641         if (lockp->readers < OSI_RWLOCK_THREADS)
642             lockp->tid[lockp->readers-1] = thrd_Current();
643 #endif
644         i = 1;
645     }
646
647     LeaveCriticalSection(csp);
648
649     if (lockOrderValidation && i) {
650         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
651         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
652         TlsSetValue(tls_LockRefH, lockRefH);
653         TlsSetValue(tls_LockRefT, lockRefT);
654     }
655
656     return i;
657 }
658
659
660 int lock_TryWrite(struct osi_rwlock *lockp)
661 {
662     long i;
663     CRITICAL_SECTION *csp;
664     osi_queue_t * lockRefH, *lockRefT;
665     osi_lock_ref_t *lockRefp;
666
667     if ((i=lockp->type) != 0)
668         if (i >= 0 && i < OSI_NLOCKTYPES)
669             return (osi_lockOps[i]->TryWriteProc)(lockp);
670
671     /* otherwise we're the fast base type */
672     csp = &osi_baseAtomicCS[lockp->atomicIndex];
673     EnterCriticalSection(csp);
674
675     if (lockOrderValidation) {
676         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
677         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
678
679         if (lockp->level != 0) {
680             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
681                 if (lockRefp->type == OSI_LOCK_RW) {
682                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
683                 }
684             }
685         }
686     }
687
688     /* here we have the fast lock, so see if we can obtain the real lock */
689     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)
690          || (lockp->readers > 0)) {
691         i = 0;
692     }
693     else {
694         /* if we're here, all clear to set the lock */
695         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
696         lockp->tid[0] = thrd_Current();
697         i = 1;
698     }
699
700     LeaveCriticalSection(csp);
701
702     if (lockOrderValidation && i) {
703         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
704         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
705         TlsSetValue(tls_LockRefH, lockRefH);
706         TlsSetValue(tls_LockRefT, lockRefT);
707     }
708
709     return i;
710 }
711
712
713 int lock_TryMutex(struct osi_mutex *lockp) {
714     long i;
715     CRITICAL_SECTION *csp;
716     osi_queue_t * lockRefH, *lockRefT;
717     osi_lock_ref_t *lockRefp;
718
719     if ((i=lockp->type) != 0)
720         if (i >= 0 && i < OSI_NLOCKTYPES)
721             return (osi_lockOps[i]->TryMutexProc)(lockp);
722
723     /* otherwise we're the fast base type */
724     csp = &osi_baseAtomicCS[lockp->atomicIndex];
725     EnterCriticalSection(csp);
726
727     if (lockOrderValidation) {
728         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
729         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
730
731         if (lockp->level != 0) {
732             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
733                 if (lockRefp->type == OSI_LOCK_MUTEX) {
734                     osi_assertx(lockRefp->mx != lockp, "Mutex already held");
735                 }
736             }
737         }
738     }
739
740     /* here we have the fast lock, so see if we can obtain the real lock */
741     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
742         i = 0;
743     }
744     else {
745         /* if we're here, all clear to set the lock */
746         _InterlockedOr(&lockp->flags, OSI_LOCKFLAG_EXCL);
747         lockp->tid = thrd_Current();
748         i = 1;
749     }
750
751     LeaveCriticalSection(csp);
752
753     if (lockOrderValidation && i) {
754         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
755         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
756         TlsSetValue(tls_LockRefH, lockRefH);
757         TlsSetValue(tls_LockRefT, lockRefT);
758     }
759
760     return i;
761 }
762
763 void osi_SleepR(LONG_PTR sleepVal, struct osi_rwlock *lockp)
764 {
765     long i;
766     CRITICAL_SECTION *csp;
767     osi_queue_t * lockRefH, *lockRefT;
768     osi_lock_ref_t *lockRefp;
769     DWORD tid = thrd_Current();
770
771     if ((i = lockp->type) != 0) {
772         if (i >= 0 && i < OSI_NLOCKTYPES)
773             (osi_lockOps[i]->SleepRProc)(sleepVal, lockp);
774         return;
775     }
776
777     /* otherwise we're the fast base type */
778     csp = &osi_baseAtomicCS[lockp->atomicIndex];
779     EnterCriticalSection(csp);
780
781     if (lockOrderValidation && lockp->level != 0) {
782         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
783         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
784
785         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
786             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
787                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
788                 lock_FreeLockRef(lockRefp);
789                 break;
790             }
791         }
792
793         TlsSetValue(tls_LockRefH, lockRefH);
794         TlsSetValue(tls_LockRefT, lockRefT);
795     }
796
797     osi_assertx(lockp->readers > 0, "osi_SleepR: not held");
798
799 #ifdef DEBUG
800     for ( i=0; i < lockp->readers && i < OSI_RWLOCK_THREADS; i++) {
801         if ( lockp->tid[i] == tid ) {
802             for ( ; i < (lockp->readers - 1) && i < (OSI_RWLOCK_THREADS - 1); i++)
803                 lockp->tid[i] = lockp->tid[i+1];
804             lockp->tid[i] = 0;
805             break;
806         }
807     }
808 #endif
809
810     /* XXX better to get the list of things to wakeup from TSignalForMLs, and
811      * then do the wakeup after SleepSpin releases the low-level mutex.
812      */
813     if (--(lockp->readers) == 0 && lockp->waiters) {
814         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
815     }
816
817     /* now call into scheduler to sleep atomically with releasing spin lock */
818     osi_SleepSpin(sleepVal, csp);
819 }
820
821 void osi_SleepW(LONG_PTR sleepVal, struct osi_rwlock *lockp)
822 {
823     long i;
824     CRITICAL_SECTION *csp;
825     osi_queue_t * lockRefH, *lockRefT;
826     osi_lock_ref_t *lockRefp;
827     DWORD tid = thrd_Current();
828
829     if ((i = lockp->type) != 0) {
830         if (i >= 0 && i < OSI_NLOCKTYPES)
831             (osi_lockOps[i]->SleepWProc)(sleepVal, lockp);
832         return;
833     }
834
835     /* otherwise we're the fast base type */
836     csp = &osi_baseAtomicCS[lockp->atomicIndex];
837     EnterCriticalSection(csp);
838
839     if (lockOrderValidation && lockp->level != 0) {
840         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
841         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
842
843         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
844             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
845                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
846                 lock_FreeLockRef(lockRefp);
847                 break;
848             }
849         }
850
851         TlsSetValue(tls_LockRefH, lockRefH);
852         TlsSetValue(tls_LockRefT, lockRefT);
853     }
854
855     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepW: not held");
856
857     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
858     lockp->tid[0] = 0;
859     if (lockp->waiters) {
860         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
861     }
862
863     /* and finally release the big lock */
864     osi_SleepSpin(sleepVal, csp);
865 }
866
867 void osi_SleepM(LONG_PTR sleepVal, struct osi_mutex *lockp)
868 {
869     long i;
870     CRITICAL_SECTION *csp;
871     osi_queue_t * lockRefH, *lockRefT;
872     osi_lock_ref_t *lockRefp;
873
874     if ((i = lockp->type) != 0) {
875         if (i >= 0 && i < OSI_NLOCKTYPES)
876             (osi_lockOps[i]->SleepMProc)(sleepVal, lockp);
877         return;
878     }
879
880     /* otherwise we're the fast base type */
881     csp = &osi_baseAtomicCS[lockp->atomicIndex];
882     EnterCriticalSection(csp);
883
884     if (lockOrderValidation && lockp->level != 0) {
885         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
886         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
887
888         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
889             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
890                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
891                 lock_FreeLockRef(lockRefp);
892                 break;
893             }
894         }
895
896         TlsSetValue(tls_LockRefH, lockRefH);
897         TlsSetValue(tls_LockRefT, lockRefT);
898     }
899
900     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepM not held");
901
902     _InterlockedAnd(&lockp->flags, ~OSI_LOCKFLAG_EXCL);
903     lockp->tid = 0;
904     if (lockp->waiters) {
905         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
906     }
907
908     /* and finally release the big lock */
909     osi_SleepSpin(sleepVal, csp);
910 }
911
912 void lock_FinalizeRWLock(osi_rwlock_t *lockp)
913 {
914     long i;
915
916     if ((i=lockp->type) != 0)
917         if (i >= 0 && i < OSI_NLOCKTYPES)
918             (osi_lockOps[i]->FinalizeRWLockProc)(lockp);
919 }
920
921 void lock_FinalizeMutex(osi_mutex_t *lockp)
922 {
923     long i;
924
925     if ((i=lockp->type) != 0)
926         if (i >= 0 && i < OSI_NLOCKTYPES)
927             (osi_lockOps[i]->FinalizeMutexProc)(lockp);
928 }
929
930 void lock_InitializeMutex(osi_mutex_t *mp, char *namep, unsigned short level)
931 {
932     int i;
933
934     if ((i = osi_lockTypeDefault) > 0) {
935         if (i >= 0 && i < OSI_NLOCKTYPES)
936             (osi_lockOps[i]->InitializeMutexProc)(mp, namep, level);
937         return;
938     }
939
940     /*
941      * otherwise we have the base case, which requires no special
942      * initialization.
943      */
944     memset(mp, 0, sizeof(osi_mutex_t));
945     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
946     mp->level = level;
947     osi_TInit(&mp->d.turn);
948     return;
949 }
950
951 void lock_InitializeRWLock(osi_rwlock_t *mp, char *namep, unsigned short level)
952 {
953     int i;
954
955     if ((i = osi_lockTypeDefault) > 0) {
956         if (i >= 0 && i < OSI_NLOCKTYPES)
957             (osi_lockOps[i]->InitializeRWLockProc)(mp, namep, level);
958         return;
959     }
960
961     /* otherwise we have the base case, which requires no special
962      * initialization.
963      */
964     memset(mp, 0, sizeof(osi_rwlock_t));
965     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
966     mp->level = level;
967     osi_TInit(&mp->d.turn);
968     return;
969 }
970
971 int lock_GetRWLockState(osi_rwlock_t *lp)
972 {
973     long i;
974     CRITICAL_SECTION *csp;
975
976     if ((i=lp->type) != 0)
977         if (i >= 0 && i < OSI_NLOCKTYPES)
978             return (osi_lockOps[i]->GetRWLockState)(lp);
979
980     /* otherwise we're the fast base type */
981     csp = &osi_baseAtomicCS[lp->atomicIndex];
982     EnterCriticalSection(csp);
983
984     /* here we have the fast lock, so see if we can obtain the real lock */
985     if (lp->flags & OSI_LOCKFLAG_EXCL)
986         i = OSI_RWLOCK_WRITEHELD;
987     else
988         i = 0;
989     if (lp->readers > 0)
990         i |= OSI_RWLOCK_READHELD;
991
992     LeaveCriticalSection(csp);
993
994     return i;
995 }
996
997 int lock_GetMutexState(struct osi_mutex *mp)
998 {
999     long i;
1000     CRITICAL_SECTION *csp;
1001
1002     if ((i=mp->type) != 0)
1003         if (i >= 0 && i < OSI_NLOCKTYPES)
1004             return (osi_lockOps[i]->GetMutexState)(mp);
1005
1006     /* otherwise we're the fast base type */
1007     csp = &osi_baseAtomicCS[mp->atomicIndex];
1008     EnterCriticalSection(csp);
1009
1010     if (mp->flags & OSI_LOCKFLAG_EXCL)
1011         i = OSI_MUTEX_HELD;
1012     else
1013         i = 0;
1014
1015     LeaveCriticalSection(csp);
1016
1017     return i;
1018 }