Windows: record history of read lock threads
[openafs.git] / src / WINNT / client_osi / osibasel.c
1 /*
2  * Copyright 2000, International Business Machines Corporation and others.
3  * All Rights Reserved.
4  *
5  * This software has been released under the terms of the IBM Public
6  * License.  For details, see the LICENSE file in the top-level source
7  * directory or online at http://www.openafs.org/dl/license10.html
8  */
9
10 /* Copyright (C) 1994 Cazamar Systems, Inc. */
11
12
13 #include <afs/param.h>
14 #include <afs/stds.h>
15
16 #include <windows.h>
17 #include "osi.h"
18 #include <assert.h>
19 #include <stdio.h>
20
21 /* atomicity-providing critical sections */
22 CRITICAL_SECTION osi_baseAtomicCS[OSI_MUTEXHASHSIZE];
23 static long     atomicIndexCounter = 0;
24
25 /* Thread local storage index for lock tracking */
26 static DWORD tls_LockRefH = 0;
27 static DWORD tls_LockRefT = 0;
28 static BOOLEAN lockOrderValidation = 0;
29 static osi_lock_ref_t * lock_ref_FreeListp = NULL;
30 static osi_lock_ref_t * lock_ref_FreeListEndp = NULL;
31 CRITICAL_SECTION lock_ref_CS;
32
33 void osi_BaseInit(void)
34 {
35     int i;
36
37     for(i=0; i<OSI_MUTEXHASHSIZE; i++)
38         InitializeCriticalSection(&osi_baseAtomicCS[i]);
39
40     if ((tls_LockRefH = TlsAlloc()) == TLS_OUT_OF_INDEXES)
41         osi_panic("TlsAlloc(tls_LockRefH) failure", __FILE__, __LINE__);
42
43     if ((tls_LockRefT = TlsAlloc()) == TLS_OUT_OF_INDEXES)
44         osi_panic("TlsAlloc(tls_LockRefT) failure", __FILE__, __LINE__);
45
46     InitializeCriticalSection(&lock_ref_CS);
47 }
48
49 void
50 osi_SetLockOrderValidation(int on)
51 {
52     lockOrderValidation = (BOOLEAN)on;
53 }
54
55 static osi_lock_ref_t *
56 lock_GetLockRef(void * lockp, char type)
57 {
58     osi_lock_ref_t * lockRefp = NULL;
59
60     EnterCriticalSection(&lock_ref_CS);
61     if (lock_ref_FreeListp) {
62         lockRefp = lock_ref_FreeListp;
63         osi_QRemoveHT( (osi_queue_t **) &lock_ref_FreeListp,
64                        (osi_queue_t **) &lock_ref_FreeListEndp,
65                        &lockRefp->q);
66     }
67     LeaveCriticalSection(&lock_ref_CS);
68
69     if (lockRefp == NULL)
70         lockRefp = (osi_lock_ref_t *)malloc(sizeof(osi_lock_ref_t));
71
72     memset(lockRefp, 0, sizeof(osi_lock_ref_t));
73     lockRefp->type = type;
74     switch (type) {
75     case OSI_LOCK_MUTEX:
76         lockRefp->mx = lockp;
77         break;
78     case OSI_LOCK_RW:
79         lockRefp->rw = lockp;
80         break;
81     default:
82         osi_panic("Invalid Lock Type", __FILE__, __LINE__);
83     }
84
85     return lockRefp;
86 }
87
88 static void
89 lock_FreeLockRef(osi_lock_ref_t * lockRefp)
90 {
91     EnterCriticalSection(&lock_ref_CS);
92     osi_QAddH( (osi_queue_t **) &lock_ref_FreeListp,
93                (osi_queue_t **) &lock_ref_FreeListEndp,
94                &lockRefp->q);
95     LeaveCriticalSection(&lock_ref_CS);
96 }
97
98 void lock_VerifyOrderRW(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_rwlock_t *lockp)
99 {
100     char msg[512];
101     osi_lock_ref_t * lockRefp;
102
103     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
104         if (lockRefp->type == OSI_LOCK_RW) {
105             if (lockRefp->rw == lockp) {
106                 sprintf(msg, "RW Lock 0x%p level %d already held", lockp, lockp->level);
107                 osi_panic(msg, __FILE__, __LINE__);
108             }
109             if (lockRefp->rw->level > lockp->level) {
110                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
111                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
112                 osi_panic(msg, __FILE__, __LINE__);
113             }
114         } else {
115             if (lockRefp->mx->level > lockp->level) {
116                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
117                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
118                 osi_panic(msg, __FILE__, __LINE__);
119             }
120             osi_assertx(lockRefp->mx->level <= lockp->level, "Lock hierarchy violation");
121         }
122     }
123 }
124
125 void lock_VerifyOrderMX(osi_queue_t *lockRefH, osi_queue_t *lockRefT, osi_mutex_t *lockp)
126 {
127     char msg[512];
128     osi_lock_ref_t * lockRefp;
129
130     for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
131         if (lockRefp->type == OSI_LOCK_MUTEX) {
132             if (lockRefp->mx == lockp) {
133                 sprintf(msg, "MX Lock 0x%p level %d already held", lockp, lockp->level);
134                 osi_panic(msg, __FILE__, __LINE__);
135             }
136             if (lockRefp->mx->level > lockp->level) {
137                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
138                          lockRefp->mx, lockRefp->mx->level, lockp, lockp->level);
139                 osi_panic(msg, __FILE__, __LINE__);
140             }
141         } else {
142             if (lockRefp->rw->level > lockp->level) {
143                 sprintf(msg, "Lock hierarchy violation Held lock 0x%p level %d > Requested lock 0x%p level %d",
144                          lockRefp->rw, lockRefp->rw->level, lockp, lockp->level);
145                 osi_panic(msg, __FILE__, __LINE__);
146             }
147         }
148     }
149 }
150
151 void lock_ObtainWrite(osi_rwlock_t *lockp)
152 {
153     long i;
154     CRITICAL_SECTION *csp;
155     osi_queue_t * lockRefH, *lockRefT;
156     osi_lock_ref_t *lockRefp;
157
158     if ((i=lockp->type) != 0) {
159         if (i >= 0 && i < OSI_NLOCKTYPES)
160             (osi_lockOps[i]->ObtainWriteProc)(lockp);
161         return;
162     }
163
164     if (lockOrderValidation) {
165         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
166         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
167
168         if (lockp->level != 0)
169             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
170     }
171
172     /* otherwise we're the fast base type */
173     csp = &osi_baseAtomicCS[lockp->atomicIndex];
174     EnterCriticalSection(csp);
175
176     /* here we have the fast lock, so see if we can obtain the real lock */
177     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL) ||
178         (lockp->readers > 0)) {
179         lockp->waiters++;
180         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp);
181         lockp->waiters--;
182         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
183     } else {
184         /* if we're here, all clear to set the lock */
185         lockp->flags |= OSI_LOCKFLAG_EXCL;
186         lockp->tid[0] = thrd_Current();
187     }
188     LeaveCriticalSection(csp);
189
190     if (lockOrderValidation) {
191         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
192         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
193         TlsSetValue(tls_LockRefH, lockRefH);
194         TlsSetValue(tls_LockRefT, lockRefT);
195     }
196 }
197
198 void lock_ObtainRead(osi_rwlock_t *lockp)
199 {
200     long i;
201     CRITICAL_SECTION *csp;
202     osi_queue_t * lockRefH, *lockRefT;
203     osi_lock_ref_t *lockRefp;
204     DWORD tid = thrd_Current();
205
206     if ((i=lockp->type) != 0) {
207         if (i >= 0 && i < OSI_NLOCKTYPES)
208             (osi_lockOps[i]->ObtainReadProc)(lockp);
209         return;
210     }
211
212     if (lockOrderValidation) {
213         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
214         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
215
216         if (lockp->level != 0)
217             lock_VerifyOrderRW(lockRefH, lockRefT, lockp);
218     }
219
220     /* otherwise we're the fast base type */
221     csp = &osi_baseAtomicCS[lockp->atomicIndex];
222     EnterCriticalSection(csp);
223
224     for ( i=0; i < lockp->readers; i++ ) {
225         osi_assertx(lockp->tid[i] != tid, "OSI_RWLOCK_READHELD");
226     }
227
228     /* here we have the fast lock, so see if we can obtain the real lock */
229     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
230         lockp->waiters++;
231         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4READ, &lockp->readers, lockp->tid, csp);
232         lockp->waiters--;
233         osi_assert(!(lockp->flags & OSI_LOCKFLAG_EXCL) && lockp->readers > 0);
234     } else {
235         /* if we're here, all clear to set the lock */
236         if (++lockp->readers <= OSI_RWLOCK_THREADS)
237             lockp->tid[lockp->readers-1] = tid;
238     }
239     LeaveCriticalSection(csp);
240
241     if (lockOrderValidation) {
242         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
243         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
244         TlsSetValue(tls_LockRefH, lockRefH);
245         TlsSetValue(tls_LockRefT, lockRefT);
246     }
247 }
248
249 void lock_ReleaseRead(osi_rwlock_t *lockp)
250 {
251     long i;
252     CRITICAL_SECTION *csp;
253     osi_queue_t * lockRefH, *lockRefT;
254     osi_lock_ref_t *lockRefp;
255     DWORD tid = thrd_Current();
256
257     if ((i = lockp->type) != 0) {
258         if (i >= 0 && i < OSI_NLOCKTYPES)
259             (osi_lockOps[i]->ReleaseReadProc)(lockp);
260         return;
261     }
262
263     if (lockOrderValidation && lockp->level != 0) {
264         int found = 0;
265         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
266         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
267
268         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
269             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
270                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
271                 lock_FreeLockRef(lockRefp);
272                 found = 1;
273                 break;
274             }
275         }
276         osi_assertx(found, "read lock not found in TLS queue");
277
278         TlsSetValue(tls_LockRefH, lockRefH);
279         TlsSetValue(tls_LockRefT, lockRefT);
280     }
281
282     /* otherwise we're the fast base type */
283     csp = &osi_baseAtomicCS[lockp->atomicIndex];
284     EnterCriticalSection(csp);
285
286     osi_assertx(lockp->readers > 0, "read lock not held");
287
288     for ( i=0; i < lockp->readers; i++) {
289         if ( lockp->tid[i] == tid ) {
290             for ( ; i < lockp->readers - 1; i++)
291                 lockp->tid[i] = lockp->tid[i+1];
292             lockp->tid[i] = 0;
293             break;
294         }
295     }
296
297     /* releasing a read lock can allow readers or writers */
298     if (--lockp->readers == 0 && !osi_TEmpty(&lockp->d.turn)) {
299         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
300     }
301     else {
302         /* and finally release the big lock */
303         LeaveCriticalSection(csp);
304     }
305 }
306
307 void lock_ReleaseWrite(osi_rwlock_t *lockp)
308 {
309     long i;
310     CRITICAL_SECTION *csp;
311     osi_queue_t * lockRefH, *lockRefT;
312     osi_lock_ref_t *lockRefp;
313
314     if ((i = lockp->type) != 0) {
315         if (i >= 0 && i < OSI_NLOCKTYPES)
316             (osi_lockOps[i]->ReleaseWriteProc)(lockp);
317         return;
318     }
319
320     if (lockOrderValidation && lockp->level != 0) {
321         int found = 0;
322         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
323         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
324
325         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
326             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
327                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
328                 lock_FreeLockRef(lockRefp);
329                 found = 1;
330                 break;
331             }
332         }
333         osi_assertx(found, "write lock not found in TLS queue");
334
335         TlsSetValue(tls_LockRefH, lockRefH);
336         TlsSetValue(tls_LockRefT, lockRefT);
337     }
338
339     /* otherwise we're the fast base type */
340     csp = &osi_baseAtomicCS[lockp->atomicIndex];
341     EnterCriticalSection(csp);
342
343     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
344     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
345
346     lockp->tid[0] = 0;
347
348     lockp->flags &= ~OSI_LOCKFLAG_EXCL;
349     if (!osi_TEmpty(&lockp->d.turn)) {
350         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
351     }
352     else {
353         /* and finally release the big lock */
354         LeaveCriticalSection(csp);
355     }
356 }
357
358 void lock_ConvertWToR(osi_rwlock_t *lockp)
359 {
360     long i;
361     CRITICAL_SECTION *csp;
362
363     if ((i = lockp->type) != 0) {
364         if (i >= 0 && i < OSI_NLOCKTYPES)
365             (osi_lockOps[i]->ConvertWToRProc)(lockp);
366         return;
367     }
368
369     /* otherwise we're the fast base type */
370     csp = &osi_baseAtomicCS[lockp->atomicIndex];
371     EnterCriticalSection(csp);
372
373     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "write lock not held");
374     osi_assertx(lockp->tid[0] == thrd_Current(), "write lock not held by current thread");
375
376     /* convert write lock to read lock */
377     lockp->flags &= ~OSI_LOCKFLAG_EXCL;
378     lockp->readers++;
379
380     if (!osi_TEmpty(&lockp->d.turn)) {
381         osi_TSignalForMLs(&lockp->d.turn, /* still have readers */ 1, csp);
382     }
383     else {
384         /* and finally release the big lock */
385         LeaveCriticalSection(csp);
386     }
387 }
388
389 void lock_ConvertRToW(osi_rwlock_t *lockp)
390 {
391     long i;
392     CRITICAL_SECTION *csp;
393     DWORD tid = thrd_Current();
394
395     if ((i = lockp->type) != 0) {
396         if (i >= 0 && i < OSI_NLOCKTYPES)
397             (osi_lockOps[i]->ConvertRToWProc)(lockp);
398         return;
399     }
400
401     /* otherwise we're the fast base type */
402     csp = &osi_baseAtomicCS[lockp->atomicIndex];
403     EnterCriticalSection(csp);
404
405     osi_assertx(!(lockp->flags & OSI_LOCKFLAG_EXCL), "write lock held");
406     osi_assertx(lockp->readers > 0, "read lock not held");
407
408     for ( i=0; i < lockp->readers; i++) {
409         if ( lockp->tid[i] == tid ) {
410             for ( ; i < lockp->readers - 1; i++)
411                 lockp->tid[i] = lockp->tid[i+1];
412             lockp->tid[i] = 0;
413             break;
414         }
415     }
416
417     if (--lockp->readers == 0) {
418         /* convert read lock to write lock */
419         lockp->flags |= OSI_LOCKFLAG_EXCL;
420         lockp->tid[0] = tid;
421     } else {
422         lockp->waiters++;
423         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, lockp->tid, csp);
424         lockp->waiters--;
425         osi_assert(lockp->readers == 0 && (lockp->flags & OSI_LOCKFLAG_EXCL));
426     }
427
428     LeaveCriticalSection(csp);
429 }
430
431 void lock_ObtainMutex(struct osi_mutex *lockp)
432 {
433     long i;
434     CRITICAL_SECTION *csp;
435     osi_queue_t * lockRefH, *lockRefT;
436     osi_lock_ref_t *lockRefp;
437
438     if ((i=lockp->type) != 0) {
439         if (i >= 0 && i < OSI_NLOCKTYPES)
440             (osi_lockOps[i]->ObtainMutexProc)(lockp);
441         return;
442     }
443
444     if (lockOrderValidation) {
445         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
446         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
447
448         if (lockp->level != 0)
449             lock_VerifyOrderMX(lockRefH, lockRefT, lockp);
450     }
451
452     /* otherwise we're the fast base type */
453     csp = &osi_baseAtomicCS[lockp->atomicIndex];
454     EnterCriticalSection(csp);
455
456     /* here we have the fast lock, so see if we can obtain the real lock */
457     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
458         lockp->waiters++;
459         osi_TWait(&lockp->d.turn, OSI_SLEEPINFO_W4WRITE, &lockp->flags, &lockp->tid, csp);
460         lockp->waiters--;
461         osi_assert(lockp->flags & OSI_LOCKFLAG_EXCL);
462     } else {
463         /* if we're here, all clear to set the lock */
464         lockp->flags |= OSI_LOCKFLAG_EXCL;
465         lockp->tid = thrd_Current();
466     }
467     LeaveCriticalSection(csp);
468
469     if (lockOrderValidation) {
470         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
471         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
472         TlsSetValue(tls_LockRefH, lockRefH);
473         TlsSetValue(tls_LockRefT, lockRefT);
474     }
475 }
476
477 void lock_ReleaseMutex(struct osi_mutex *lockp)
478 {
479     long i;
480     CRITICAL_SECTION *csp;
481     osi_queue_t * lockRefH, *lockRefT;
482     osi_lock_ref_t *lockRefp;
483
484     if ((i = lockp->type) != 0) {
485         if (i >= 0 && i < OSI_NLOCKTYPES)
486             (osi_lockOps[i]->ReleaseMutexProc)(lockp);
487         return;
488     }
489
490     if (lockOrderValidation && lockp->level != 0) {
491         int found = 0;
492         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
493         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
494
495         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
496             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
497                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
498                 lock_FreeLockRef(lockRefp);
499                 found = 1;
500                 break;
501             }
502         }
503
504         osi_assertx(found, "mutex lock not found in TLS queue");
505         TlsSetValue(tls_LockRefH, lockRefH);
506         TlsSetValue(tls_LockRefT, lockRefT);
507     }
508
509     /* otherwise we're the fast base type */
510     csp = &osi_baseAtomicCS[lockp->atomicIndex];
511     EnterCriticalSection(csp);
512
513     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "mutex not held");
514     osi_assertx(lockp->tid == thrd_Current(), "mutex not held by current thread");
515
516     lockp->flags &= ~OSI_LOCKFLAG_EXCL;
517     lockp->tid = 0;
518     if (!osi_TEmpty(&lockp->d.turn)) {
519         osi_TSignalForMLs(&lockp->d.turn, 0, csp);
520     }
521     else {
522         /* and finally release the big lock */
523         LeaveCriticalSection(csp);
524     }
525 }
526
527 int lock_TryRead(struct osi_rwlock *lockp)
528 {
529     long i;
530     CRITICAL_SECTION *csp;
531     osi_queue_t * lockRefH, *lockRefT;
532     osi_lock_ref_t *lockRefp;
533
534     if ((i=lockp->type) != 0)
535         if (i >= 0 && i < OSI_NLOCKTYPES)
536             return (osi_lockOps[i]->TryReadProc)(lockp);
537
538     if (lockOrderValidation) {
539         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
540         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
541
542         if (lockp->level != 0) {
543             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
544                 if (lockRefp->type == OSI_LOCK_RW) {
545                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
546                 }
547             }
548         }
549     }
550
551     /* otherwise we're the fast base type */
552     csp = &osi_baseAtomicCS[lockp->atomicIndex];
553     EnterCriticalSection(csp);
554
555     /* here we have the fast lock, so see if we can obtain the real lock */
556     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
557         i = 0;
558     }
559     else {
560         /* if we're here, all clear to set the lock */
561         if (++lockp->readers < OSI_RWLOCK_THREADS)
562             lockp->tid[lockp->readers-1] = thrd_Current();
563         i = 1;
564     }
565
566     LeaveCriticalSection(csp);
567
568     if (lockOrderValidation && i) {
569         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
570         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
571         TlsSetValue(tls_LockRefH, lockRefH);
572         TlsSetValue(tls_LockRefT, lockRefT);
573     }
574
575     return i;
576 }
577
578
579 int lock_TryWrite(struct osi_rwlock *lockp)
580 {
581     long i;
582     CRITICAL_SECTION *csp;
583     osi_queue_t * lockRefH, *lockRefT;
584     osi_lock_ref_t *lockRefp;
585
586     if ((i=lockp->type) != 0)
587         if (i >= 0 && i < OSI_NLOCKTYPES)
588             return (osi_lockOps[i]->TryWriteProc)(lockp);
589
590     if (lockOrderValidation) {
591         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
592         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
593
594         if (lockp->level != 0) {
595             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
596                 if (lockRefp->type == OSI_LOCK_RW) {
597                     osi_assertx(lockRefp->rw != lockp, "RW Lock already held");
598                 }
599             }
600         }
601     }
602
603     /* otherwise we're the fast base type */
604     csp = &osi_baseAtomicCS[lockp->atomicIndex];
605     EnterCriticalSection(csp);
606
607     /* here we have the fast lock, so see if we can obtain the real lock */
608     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)
609          || (lockp->readers > 0)) {
610         i = 0;
611     }
612     else {
613         /* if we're here, all clear to set the lock */
614         lockp->flags |= OSI_LOCKFLAG_EXCL;
615         lockp->tid[0] = thrd_Current();
616         i = 1;
617     }
618
619     LeaveCriticalSection(csp);
620
621     if (lockOrderValidation && i) {
622         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_RW);
623         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
624         TlsSetValue(tls_LockRefH, lockRefH);
625         TlsSetValue(tls_LockRefT, lockRefT);
626     }
627
628     return i;
629 }
630
631
632 int lock_TryMutex(struct osi_mutex *lockp) {
633     long i;
634     CRITICAL_SECTION *csp;
635     osi_queue_t * lockRefH, *lockRefT;
636     osi_lock_ref_t *lockRefp;
637
638     if ((i=lockp->type) != 0)
639         if (i >= 0 && i < OSI_NLOCKTYPES)
640             return (osi_lockOps[i]->TryMutexProc)(lockp);
641
642     if (lockOrderValidation) {
643         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
644         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
645
646         if (lockp->level != 0) {
647             for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
648                 if (lockRefp->type == OSI_LOCK_MUTEX) {
649                     osi_assertx(lockRefp->mx != lockp, "Mutex already held");
650                 }
651             }
652         }
653     }
654
655     /* otherwise we're the fast base type */
656     csp = &osi_baseAtomicCS[lockp->atomicIndex];
657     EnterCriticalSection(csp);
658
659     /* here we have the fast lock, so see if we can obtain the real lock */
660     if (lockp->waiters > 0 || (lockp->flags & OSI_LOCKFLAG_EXCL)) {
661         i = 0;
662     }
663     else {
664         /* if we're here, all clear to set the lock */
665         lockp->flags |= OSI_LOCKFLAG_EXCL;
666         lockp->tid = thrd_Current();
667         i = 1;
668     }
669
670     LeaveCriticalSection(csp);
671
672     if (lockOrderValidation && i) {
673         lockRefp = lock_GetLockRef(lockp, OSI_LOCK_MUTEX);
674         osi_QAddH(&lockRefH, &lockRefT, &lockRefp->q);
675         TlsSetValue(tls_LockRefH, lockRefH);
676         TlsSetValue(tls_LockRefT, lockRefT);
677     }
678     return i;
679 }
680
681 void osi_SleepR(LONG_PTR sleepVal, struct osi_rwlock *lockp)
682 {
683     long i;
684     CRITICAL_SECTION *csp;
685     osi_queue_t * lockRefH, *lockRefT;
686     osi_lock_ref_t *lockRefp;
687     DWORD tid = thrd_Current();
688
689     if ((i = lockp->type) != 0) {
690         if (i >= 0 && i < OSI_NLOCKTYPES)
691             (osi_lockOps[i]->SleepRProc)(sleepVal, lockp);
692         return;
693     }
694
695     if (lockOrderValidation && lockp->level != 0) {
696         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
697         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
698
699         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
700             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
701                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
702                 lock_FreeLockRef(lockRefp);
703                 break;
704             }
705         }
706
707         TlsSetValue(tls_LockRefH, lockRefH);
708         TlsSetValue(tls_LockRefT, lockRefT);
709     }
710
711     /* otherwise we're the fast base type */
712     csp = &osi_baseAtomicCS[lockp->atomicIndex];
713     EnterCriticalSection(csp);
714
715     osi_assertx(lockp->readers > 0, "osi_SleepR: not held");
716
717     for ( i=0; i < lockp->readers; i++) {
718         if ( lockp->tid[i] == tid ) {
719             for ( ; i < lockp->readers - 1; i++)
720                 lockp->tid[i] = lockp->tid[i+1];
721             lockp->tid[i] = 0;
722             break;
723         }
724     }
725
726     /* XXX better to get the list of things to wakeup from TSignalForMLs, and
727      * then do the wakeup after SleepSpin releases the low-level mutex.
728      */
729     if (--lockp->readers == 0 && !osi_TEmpty(&lockp->d.turn)) {
730         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
731     }
732
733     /* now call into scheduler to sleep atomically with releasing spin lock */
734     osi_SleepSpin(sleepVal, csp);
735 }
736
737 void osi_SleepW(LONG_PTR sleepVal, struct osi_rwlock *lockp)
738 {
739     long i;
740     CRITICAL_SECTION *csp;
741     osi_queue_t * lockRefH, *lockRefT;
742     osi_lock_ref_t *lockRefp;
743     DWORD tid = thrd_Current();
744
745     if ((i = lockp->type) != 0) {
746         if (i >= 0 && i < OSI_NLOCKTYPES)
747             (osi_lockOps[i]->SleepWProc)(sleepVal, lockp);
748         return;
749     }
750
751     if (lockOrderValidation && lockp->level != 0) {
752         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
753         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
754
755         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
756             if (lockRefp->type == OSI_LOCK_RW && lockRefp->rw == lockp) {
757                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
758                 lock_FreeLockRef(lockRefp);
759                 break;
760             }
761         }
762
763         TlsSetValue(tls_LockRefH, lockRefH);
764         TlsSetValue(tls_LockRefT, lockRefT);
765     }
766
767     /* otherwise we're the fast base type */
768     csp = &osi_baseAtomicCS[lockp->atomicIndex];
769     EnterCriticalSection(csp);
770
771     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepW: not held");
772
773     lockp->flags &= ~OSI_LOCKFLAG_EXCL;
774     lockp->tid[0] = 0;
775     if (!osi_TEmpty(&lockp->d.turn)) {
776         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
777     }
778
779     /* and finally release the big lock */
780     osi_SleepSpin(sleepVal, csp);
781 }
782
783 void osi_SleepM(LONG_PTR sleepVal, struct osi_mutex *lockp)
784 {
785     long i;
786     CRITICAL_SECTION *csp;
787     osi_queue_t * lockRefH, *lockRefT;
788     osi_lock_ref_t *lockRefp;
789
790     if ((i = lockp->type) != 0) {
791         if (i >= 0 && i < OSI_NLOCKTYPES)
792             (osi_lockOps[i]->SleepMProc)(sleepVal, lockp);
793         return;
794     }
795
796     if (lockOrderValidation && lockp->level != 0) {
797         lockRefH = (osi_queue_t *)TlsGetValue(tls_LockRefH);
798         lockRefT = (osi_queue_t *)TlsGetValue(tls_LockRefT);
799
800         for (lockRefp = (osi_lock_ref_t *)lockRefH ; lockRefp; lockRefp = (osi_lock_ref_t *)osi_QNext(&lockRefp->q)) {
801             if (lockRefp->type == OSI_LOCK_MUTEX && lockRefp->mx == lockp) {
802                 osi_QRemoveHT(&lockRefH, &lockRefT, &lockRefp->q);
803                 lock_FreeLockRef(lockRefp);
804                 break;
805             }
806         }
807
808         TlsSetValue(tls_LockRefH, lockRefH);
809         TlsSetValue(tls_LockRefT, lockRefT);
810     }
811
812     /* otherwise we're the fast base type */
813     csp = &osi_baseAtomicCS[lockp->atomicIndex];
814     EnterCriticalSection(csp);
815
816     osi_assertx(lockp->flags & OSI_LOCKFLAG_EXCL, "osi_SleepM not held");
817
818     lockp->flags &= ~OSI_LOCKFLAG_EXCL;
819     lockp->tid = 0;
820     if (!osi_TEmpty(&lockp->d.turn)) {
821         osi_TSignalForMLs(&lockp->d.turn, 0, NULL);
822     }
823
824     /* and finally release the big lock */
825     osi_SleepSpin(sleepVal, csp);
826 }
827
828 void lock_FinalizeRWLock(osi_rwlock_t *lockp)
829 {
830     long i;
831
832     if ((i=lockp->type) != 0)
833         if (i >= 0 && i < OSI_NLOCKTYPES)
834             (osi_lockOps[i]->FinalizeRWLockProc)(lockp);
835 }
836
837 void lock_FinalizeMutex(osi_mutex_t *lockp)
838 {
839     long i;
840
841     if ((i=lockp->type) != 0)
842         if (i >= 0 && i < OSI_NLOCKTYPES)
843             (osi_lockOps[i]->FinalizeMutexProc)(lockp);
844 }
845
846 void lock_InitializeMutex(osi_mutex_t *mp, char *namep, unsigned short level)
847 {
848     int i;
849
850     if ((i = osi_lockTypeDefault) > 0) {
851         if (i >= 0 && i < OSI_NLOCKTYPES)
852             (osi_lockOps[i]->InitializeMutexProc)(mp, namep, level);
853         return;
854     }
855
856     /* otherwise we have the base case, which requires no special
857      * initialization.
858      */
859     mp->type = 0;
860     mp->flags = 0;
861     mp->tid = 0;
862     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
863     mp->level = level;
864     osi_TInit(&mp->d.turn);
865     return;
866 }
867
868 void lock_InitializeRWLock(osi_rwlock_t *mp, char *namep, unsigned short level)
869 {
870     int i;
871
872     if ((i = osi_lockTypeDefault) > 0) {
873         if (i >= 0 && i < OSI_NLOCKTYPES)
874             (osi_lockOps[i]->InitializeRWLockProc)(mp, namep, level);
875         return;
876     }
877
878     /* otherwise we have the base case, which requires no special
879      * initialization.
880      */
881     memset(mp, 0, sizeof(osi_rwlock_t));
882     mp->atomicIndex = (unsigned short)(InterlockedIncrement(&atomicIndexCounter) % OSI_MUTEXHASHSIZE);
883     mp->level = level;
884     osi_TInit(&mp->d.turn);
885     return;
886 }
887
888 int lock_GetRWLockState(osi_rwlock_t *lp)
889 {
890     long i;
891     CRITICAL_SECTION *csp;
892
893     if ((i=lp->type) != 0)
894         if (i >= 0 && i < OSI_NLOCKTYPES)
895             return (osi_lockOps[i]->GetRWLockState)(lp);
896
897     /* otherwise we're the fast base type */
898     csp = &osi_baseAtomicCS[lp->atomicIndex];
899     EnterCriticalSection(csp);
900
901     /* here we have the fast lock, so see if we can obtain the real lock */
902     if (lp->flags & OSI_LOCKFLAG_EXCL)
903         i = OSI_RWLOCK_WRITEHELD;
904     else
905         i = 0;
906     if (lp->readers > 0)
907         i |= OSI_RWLOCK_READHELD;
908
909     LeaveCriticalSection(csp);
910
911     return i;
912 }
913
914 int lock_GetMutexState(struct osi_mutex *mp)
915 {
916     long i;
917     CRITICAL_SECTION *csp;
918
919     if ((i=mp->type) != 0)
920         if (i >= 0 && i < OSI_NLOCKTYPES)
921             return (osi_lockOps[i]->GetMutexState)(mp);
922
923     /* otherwise we're the fast base type */
924     csp = &osi_baseAtomicCS[mp->atomicIndex];
925     EnterCriticalSection(csp);
926
927     if (mp->flags & OSI_LOCKFLAG_EXCL)
928         i = OSI_MUTEX_HELD;
929     else
930         i = 0;
931
932     LeaveCriticalSection(csp);
933
934     return i;
935 }