forked from andremussche/scalemm
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathScaleMM2.pas
1252 lines (1108 loc) · 36.7 KB
/
ScaleMM2.pas
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// fast scaling memory manager for Delphi
// licensed under a MPL/GPL/LGPL tri-license; version 0.1
unit ScaleMM2;
{$Include smmOptions.inc}
{
Fast Scaling Memory Manager 2.4 for Delphi
Description:
Simple, small and compact Memory Manager. Architectured in order to scale on
multi core CPU's (which is what FastMM4 is lacking).
Homepage:
http://code.google.com/p/scalemm
by André Mussche ([email protected])
Usage:
- Place ScaleMM2 as the very first unit under the "uses" clause of your
project's .dpr file.
License:
Released under Mozilla Public License 1.1
Modifications by A.Bouchez - http://synopse.info:
- Compiles from Delphi 6 up to Delphi XE;
- Some pascal code converted to faster asm;
- Some code refactoring, a lot of comments added;
- Added medium block handling from 2048 bytes up to 16384;
- Released under MPL 1.1/GPL 2.0/LGPL 2.1 tri-license.
*** BEGIN LICENSE BLOCK *****
Version: MPL 1.1/GPL 2.0/LGPL 2.1
The contents of this file are subject to the Mozilla Public License Version
1.1 (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.mozilla.org/MPL
Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
for the specific language governing rights and limitations under the License.
Portions created by the Initial Developer are Copyright (C) 2010
the Initial Developer. All Rights Reserved.
Contributor(s):
- Arnaud Bouchez http://synopse.info
Portions created by each contributor are Copyright (C) 2010
each contributor. All Rights Reserved.
Alternatively, the contents of this file may be used under the terms of
either the GNU General Public License Version 2 or later (the "GPL"), or
the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
in which case the provisions of the GPL or the LGPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of either the GPL or the LGPL, and not to allow others to
use your version of this file under the terms of the MPL, indicate your
decision by deleting the provisions above and replace them with the notice
and other provisions required by the GPL or the LGPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the MPL, the GPL or the LGPL.
***** END LICENSE BLOCK *****
Change log:
Version 1.0 (3 December 2010):
- First stable version
Version 1.1 (6 December 2010):
- Some optimizations for better "Fast Code MM Challenge Benchmark" results
(lower memory overhead, more memory reuse, less locking)
Version 2.0.2a (25 Januari 2011):
- added medium memory handling (<1Mb), large memory is direct done via VirtualAlloc etc
- splitted in seperate units to make developing/testing easier
- empty units for stats and logging (will be implemented later)
Version 2.0.4b (23 Februari 2011):
- realloc optimizations
- lots of internal CheckMem's (for validation)
- interthread memory is now handled (alloc in thread 1, free in thread 2)
- small (difficult to find) bugs fixed and other optimalizations
- check for 8byte alignment (needed for OmniThreadLibrary etc)
Version 2.0.5 (19 March 2011):
- small size realloc bug fixed, now passes FastCode validations (D2007 and D2010)
Version 2.1 (06 March 2012):
- small bugs fixed
- many additional checks added
- interthread memory finally stable
Note: can leak memory (or have increased mem usage) over time
Version 2.1.1 (08 March 2012):
- 64bit version (Delphi XE2)
Version 2.1.2 (13 March 2012):
- Initial code for releasing (and checking) all mem at shutdown
- fixed big mem leak in small memory manager (interthread mem was never freed)
- optimization for interthread memory in small mem manager (lock-free with CAS)
Version 2.1.3 (28 March 2012):
- shared memory implementation (thanks to FastMM for the code, Maxx xxaM for testing)
- Realloc bug fix with "large mem" (thanks to Maxx xxaM)
Version 2.1.4 (21 May 2012):
- fixed issue 6: AV when using SetLocaleOverride (bug in DXE2?) (thanks to Maxx xxaM)
Version 2.1.5 (27 July till 27 Sept 2012), thanks to Maxx xxaM:
- 64bit issues fixed
- inplace expanded virtualmem (realloc) was not properly freed
- realloc of large memory, size was increased twice with 25% (=50%)
- memleak + CAS hang (due to integer overflow)
- large interthread memory was not correctly freed
Version 2.1.6 (9-3-2013), thanks to Maxx xxaM and Thiago:
- 64bit wrong alignment can give AV's (issue 11)
Version 2.2 (6-8-2013)
- make it possible to use more than 2gb in 32bit (thanks to Maxx xxaM)
{$SetPEFlags IMAGE_FILE_LARGE_ADDRESS_AWARE
- rare AV with interthread (small) memory and heavy load (pdeusp)
Version 2.3 (22-10-2013)
- interthread memory was not always released to Windows, giving out of memory (thanks to Maxx xxaM)
- AV when doing realloc or free on nil pointer (thanks to qiu.songlin)
- fixed issue 16: alloc of large 2 dimensional array was getting slower and slower (thanks to tf.rangel)
Version 2.4 (12-12-2013)
- interthread memory fixes (stability)
Version 2.4.1 (16-12-2013)
- optimize.move included (much faster Move() function due to SSE3)
Version 2.5 (10-11-2015)
- rare AV fixed in 64bit with high load due to misalignment
Version 2.5.1 (16-3-2016)
- rare AV fixed in TGlobalMemManager.GetNewThreadManager (thanks to Molnár Attila)
Version 2.6 (27-4-2016)
- increasing memory usage with lot of short living threads (DataSnap, Indy) (thanks to Hans Wendel)
}
interface
uses
smmStatistics, smmLogging,
smmTypes,
smmSmallMemory, smmMediumMemory, smmLargeMemory;
type
PThreadMemManager = ^TThreadMemManager;
/// handles per-thread memory managment
TThreadMemManager = object
public
/// link to the list of mem freed in other thread
FOtherThreadFreedMemory: PBaseFreeMemHeader;
FOtherThreadFreeLock: NativeUInt;
FOtherThreadFreeLockRecursion: NativeUInt;
function TryLock: boolean;
procedure Lock;
procedure UnLock;
public
FThreadId: NativeUint;
FThreadTerminated: Boolean; //is this thread memory available to new thread?
// link to list of items to reuse after thread terminated
FNextThreadManager: PThreadMemManager;
FNextFreeThreadManager: PThreadMemManager;
//procedure AddFreeMemToOwnerThread(aFirstMem, aLastMem: PBaseFreeMemHeader);
public
{$IFDEF Align8Bytes}
{$ifndef CPUX64} //32bit
Filer1: Int32;
{$endif}
{$ENDIF}
{$IFDEF Align16Bytes}
{$ifndef CPUX64} //32bit
Filer1: Pointer;
//Filer2: Pointer;
{$else CPUX64} //64bit
Filer1: Pointer;
//Filer2: Pointer;
{$endif}
{$ENDIF}
FSmallMemManager : TSmallMemThreadManager;
FMediumMemManager: TMediumThreadManager;
FLargeMemManager : TLargeMemThreadManager;
protected
{$IFDEF SCALEMM_STATISTICS}
FStatistic: TMemoryStatistics;
{$ENDIF}
{$IFDEF SCALEMM_LOGGING}
FLogging: TMemoryLogging;
{$ENDIF}
{$IFDEF Align8Bytes}
{$ifndef CPUX64} //32bit
Filer_1: Int32;
{$endif}
{$ENDIF}
{$IFDEF Align16Bytes}
{$ifndef CPUX64} //32bit
Filer_1: Pointer;
{$else CPUX64} //64bit
Filer_1: Pointer;
{$endif}
{$ENDIF}
protected
procedure FreeMemOfOtherThread(aMemory: PBaseMemHeader);
function ReallocMemOfOtherThread(aMemory: Pointer; aSize: NativeUInt): Pointer;
function FreeMemFromOtherThread(aMemory: PBaseMemHeader): NativeInt;
public
procedure Init;
procedure Reset;
procedure ReleaseAllFreeMem;
procedure CheckMem(aMemory: Pointer);
procedure DumpToFile(aFile: THandle; aTotalStats, aSingleStats: PThreadMemManagerStats);
function IsMemoryFromOtherThreadsPresent: Boolean;
procedure ProcessFreedMemFromOtherThreads(aSkipSmall: boolean);
function GetMem(aSize: NativeInt) : Pointer; {$ifdef HASINLINE}inline;{$ENDIF}
function FreeMem(aMemory: Pointer): NativeInt; {$ifdef HASINLINE}inline;{$ENDIF}
function ReallocMem(aMemory: Pointer; aSize: NativeUInt): Pointer; {$ifdef HASINLINE}inline;{$ENDIF}
end;
{$if CompilerVersion >= 23} //Delphi XE2
function Scale_GetMem(aSize: NativeInt) : Pointer;
function Scale_AllocMem(aSize: NativeInt): Pointer;
function Scale_ReallocMem(aMemory: Pointer; aSize: NativeInt): Pointer;
{$else}
function Scale_GetMem(aSize: Integer) : Pointer;
function Scale_AllocMem(aSize: Cardinal): Pointer;
function Scale_ReallocMem(aMemory: Pointer; aSize: Integer): Pointer;
{$ifend}
function Scale_FreeMem(aMemory: Pointer): Integer;
procedure Scale_CheckMem(aMemory: Pointer);
function GetThreadMemManager: PThreadMemManager;
function CreateMemoryManager: PThreadMemManager;
procedure ScaleMMInstall;
procedure ScaleMMUnInstall;
{$IFDEF PURE_PASCAL}
threadvar
GCurrentThreadManager: PThreadMemManager;
{$ENDIF}
implementation
// Windows.pas unit dependency should be not used -> seperate file
uses
{$IFDEF CPUX86}
Optimize.Move,
{$ENDIF}
smmFunctions, smmGlobal;
{$IFDEF PURE_PASCAL}
function GetThreadMemManager: PThreadMemManager; {$ifdef HASINLINE}inline;{$ENDIF}
begin
Result := GCurrentThreadManager;
if Result = nil then
begin
Result := CreateMemoryManager;
Assert(not Result.FThreadTerminated);
Assert(not Result.FSmallMemManager.OwnerThread.FThreadTerminated);
end;
if Result.FThreadTerminated then
//Assert(Result.FThreadId = 1);
else
begin
Assert(Result.FThreadId = GetCurrentThreadId);
Assert(Result.FSmallMemManager.OwnerThread.FThreadId = GetCurrentThreadId);
end;
Assert(Result.FSmallMemManager.OwnerThread = PBaseThreadManager(Result));
Assert(Result.FMediumMemManager.OwnerThread = PBaseThreadManager(Result));
end;
{$ELSE}
var
GOwnTlsIndex,
GOwnTlsOffset: NativeUInt;
function GetThreadMemManager: PThreadMemManager;
asm
{$IFDEF SCALE_INJECT_OFFSET}
mov eax,123456789 // dummy value: calc once and inject at runtime
{$ELSE}
mov eax,GOwnTlsOffset // 2% slower, so we default use injected offset
{$ENDIF}
mov ecx,fs:[$00000018]
mov eax,[ecx+eax] // fixed offset, calculated only once
or eax,eax
jz CreateMemoryManager
end;
procedure _FixedOffset;
{$IFDEF SCALE_INJECT_OFFSET}
var p: PAnsiChar;
{$ENDIF}
begin
GOwnTlsOffset := GOwnTlsIndex * 4 + $0e10;
{$IFDEF SCALE_INJECT_OFFSET}
p := @GetThreadMemManager;
SetPermission(p, 5, PAGE_EXECUTE_READWRITE);
PCardinal(p+1)^ := GOwnTlsOffset; // write fixed offset
{$ENDIF}
end;
{$ENDIF PURE_PASCAL}
function CreateMemoryManager: PThreadMemManager;
begin
Result := smmGlobal.GlobalManager.GetNewThreadManager;
// if Result = nil then
// begin
// Result := VirtualAlloc( nil,
// 64 * 1024,
// SizeOf(TThreadMemManager),
// MEM_COMMIT {$ifdef AlwaysAllocateTopDown} or MEM_TOP_DOWN{$endif},
// PAGE_READWRITE);
// Result.Init;
// end
// else
begin
Result.FThreadId := GetCurrentThreadId;
Result.FThreadTerminated := False;
end;
{$IFDEF SCALEMM_DEBUG}
Result.CheckMem(nil);
{$ENDIF}
{$IFDEF PURE_PASCAL}
GCurrentThreadManager := Result;
{$ELSE}
TlsSetValue(GOwnTLSIndex, Result);
{$ENDIF}
end;
{ TThreadMemManager }
procedure TThreadMemManager.ProcessFreedMemFromOtherThreads(aSkipSmall: boolean);
var
pcurrentmem, ptempmem: PBaseFreeMemHeader;
begin
if FOtherThreadFreedMemory = nil then Exit;
//Assert(Self.FThreadId > 1);
//LOCK
if not TryLock then Exit;
if not aSkipSmall and FSmallMemManager.IsMemoryFromOtherThreadsPresent then
FSmallMemManager.FreeThreadFreedMem;
pcurrentmem := FOtherThreadFreedMemory;
FOtherThreadFreedMemory := nil;
//UNLOCK
UnLock;
//free all mem in linked list
while pcurrentmem <> nil do
begin
ptempmem := pcurrentmem;
pcurrentmem := pcurrentmem.NextThreadFree;
//free
Self.FreeMemFromOtherThread( PBaseMemHeader(ptempmem) );
end;
end;
function TThreadMemManager.ReallocMem(aMemory: Pointer;
aSize: NativeUInt): Pointer;
var
pm: PBaseMemHeader;
ot: PBaseSizeManager;
begin
if FOtherThreadFreedMemory <> nil then
ProcessFreedMemFromOtherThreads(True);
pm := PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader));
//check realloc of freed mem
if (pm.Size and 1 = 0) then //not free?
begin
//medium+large mem has ownerthread instead of ownerblock (optimization)
if NativeUInt(pm.OwnerBlock) and 3 <> 0 then
begin
//other thread?
if PThreadMemManager( NativeUInt(pm.OwnerBlock) and -4) <> @Self then
begin
Result := ReallocMemOfOtherThread(aMemory, aSize);
Exit;
end;
//large or medium?
if NativeUInt(pm.OwnerBlock) and 2 = 0 then
Result := FMediumMemManager.ReallocMem(aMemory, aSize)
else
Result := FLargeMemManager.ReallocMemWithHeader(aMemory, aSize)
end
else
//small mem
begin
ot := pm.OwnerBlock.OwnerManager;
if ot = @FSmallMemManager then
Result := FSmallMemManager.ReallocMem(aMemory, aSize)
else
Result := ReallocMemOfOtherThread(aMemory, aSize);
end
end
else
begin
Result := nil;
Error(reInvalidPtr); //double free?
end;
{$IFDEF SCALEMM_DEBUG}
CheckMem(nil);
{$ENDIF}
end;
function TThreadMemManager.ReallocMemOfOtherThread(aMemory: Pointer;
aSize: NativeUInt): Pointer;
var
pm: PBaseMemHeader;
begin
Result := Self.GetMem(aSize);
pm := PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader));
if aSize > pm.Size then
Move(aMemory^, Result^, pm.Size) // copy (use smaller old size)
else
Move(aMemory^, Result^, aSize); // copy (use smaller new size)
Self.FreeMemOfOtherThread(pm);
end;
procedure TThreadMemManager.ReleaseAllFreeMem;
begin
ProcessFreedMemFromOtherThreads(False);
FSmallMemManager.ReleaseAllFreeMem;
FMediumMemManager.ReleaseAllFreeMem;
end;
procedure TThreadMemManager.Reset;
begin
FThreadId := 0;
FThreadTerminated := True;
//FOtherThreadFreedMemory := nil;
FNextFreeThreadManager := nil;
FSmallMemManager.Reset;
FMediumMemManager.Reset;
end;
function TThreadMemManager.TryLock: boolean;
var
iCurrentThreadId: NativeUInt;
begin
iCurrentThreadId := GetCurrentThreadId;
if (FOtherThreadFreeLock = iCurrentThreadId) and
(FOtherThreadFreeLockRecursion > 0) then
begin
Assert( CAS32(iCurrentThreadId, iCurrentThreadId, @FOtherThreadFreeLock) );
inc(FOtherThreadFreeLockRecursion);
Result := True;
Exit;
end;
//LOCK: no threads may be removed/freed now
Result := CAS32(0, iCurrentThreadId, @FOtherThreadFreeLock);
if Result then
inc(FOtherThreadFreeLockRecursion);
//Result := CAS32(0, 1, @FOtherThreadFreeLock);
end;
procedure TThreadMemManager.UnLock;
begin
//if not CAS32(1, 0, @FOtherThreadFreeLock) then
// Assert(False);
//FOtherThreadFreeLock := False;
dec(FOtherThreadFreeLockRecursion);
if FOtherThreadFreeLockRecursion = 0 then
FOtherThreadFreeLock := 0;
end;
procedure TThreadMemManager.Lock;
var
iCurrentThreadId: NativeUInt;
begin
iCurrentThreadId := GetCurrentThreadId;
if (FOtherThreadFreeLock = iCurrentThreadId) and
(FOtherThreadFreeLockRecursion > 0) then
begin
Assert( CAS32(iCurrentThreadId, iCurrentThreadId, @FOtherThreadFreeLock) );
inc(FOtherThreadFreeLockRecursion);
Exit;
end;
//LOCK: no threads may be removed/freed now
while not CAS32(0, iCurrentThreadId, @FOtherThreadFreeLock) do
begin
//small wait: try to swith to other pending thread (if any) else direct continue
if not SwitchToThread then
sleep(0);
//try again
if CAS32(0, iCurrentThreadId, @FOtherThreadFreeLock) then
Break;
//wait some longer: force swith to any other thread
sleep(1);
end;
inc(FOtherThreadFreeLockRecursion);
{
//unlock
repeat
if CAS32(0, 1, @FOtherThreadFreeLock) then
Break;
//small wait: try to swith to other pending thread (if any) else direct continue
if not SwitchToThread then
sleep(0);
//try again
if CAS32(0, 1, @FOtherThreadFreeLock) then
Break;
//wait some longer: force swith to any other thread
sleep(1);
until False;
}
end;
procedure TThreadMemManager.FreeMemOfOtherThread(aMemory: PBaseMemHeader);
var
p: Pointer;
pm: PMediumHeader;
begin
//large mem can be direct freed
if NativeUInt(aMemory.OwnerBlock) and 2 <> 0 then
//if aMemory.OwnerBlock.OwnerThread.SizeType = stLarge then
begin
//convert to "client" pointer again to be able to use the normal functions
p := Pointer(NativeUInt(aMemory) + SizeOf(TBaseMemHeader));
FLargeMemManager.FreeMemWithHeader(p);
Exit;
end
//medium mem
else if NativeUInt(aMemory.OwnerBlock) and 3 <> 0 then
begin
pm := PMediumHeader( NativeUInt(aMemory) + SizeOf(TBaseMemHeader) - SizeOf(TMediumHeader));
pm.ThreadFree;
end
//small mem
else
PSmallMemHeader(aMemory).OwnerBlock.ThreadFreeMem(PSmallMemHeader(aMemory));
end;
procedure TThreadMemManager.CheckMem(aMemory: Pointer);
var
pm: PBaseMemHeader;
ot: PBaseSizeManager;
tm: PThreadMemManager;
begin
if aMemory = nil then
begin
FSmallMemManager.CheckAllMem;
FMediumMemManager.CheckMem(nil);
Exit;
end;
Assert(aMemory <> nil);
pm := PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader));
Assert(pm.OwnerBlock <> nil);
//medium or large mem?
if NativeUInt(pm.OwnerBlock) and 3 <> 0 then
begin
//other thread?
tm := PThreadMemManager( NativeUInt(pm.OwnerBlock) and -4);
if tm <> @Self then
Exit; //cannot check mem of other thread!
Assert(tm <> nil);
//large or medium?
if NativeUInt(pm.OwnerBlock) and 2 = 0 then
tm.FMediumMemManager.CheckMem(aMemory)
else
tm.FLargeMemManager.CheckMem(aMemory);
end
else
//small mem
begin
ot := pm.OwnerBlock.OwnerManager;
PThreadMemManager(ot.OwnerThread).FSmallMemManager.CheckMem(aMemory);
end;
end;
procedure TThreadMemManager.DumpToFile(aFile: THandle; aTotalStats,
aSingleStats: PThreadMemManagerStats);
begin
FSmallMemManager.DumpToFile(aFile, aTotalStats, aSingleStats);
FMediumMemManager.DumpToFile(aFile, aTotalStats, aSingleStats);
FLargeMemManager.DumpToFile(aFile, aTotalStats, aSingleStats);
end;
function TThreadMemManager.FreeMem(aMemory: Pointer): NativeInt;
var
pm: PBaseMemHeader;
ot: PBaseSizeManager;
pt: PThreadMemManager;
begin
//AV when doing realloc or free on nil pointer, thanks to qiu.songlin
if aMemory = nil then
begin
Result := 0;
Exit;
end;
if FOtherThreadFreedMemory <> nil then
ProcessFreedMemFromOtherThreads(True);
pm := PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader));
//check double free
if (pm.Size and 1 <> 0) then
Error(reInvalidPtr);
//medium or large mem?
if NativeUInt(pm.OwnerBlock) and 3 <> 0 then
begin
pt := PThreadMemManager( NativeUInt(pm.OwnerBlock) and -4);
if pt <> @Self then
//other thread?
begin
FreeMemOfOtherThread(pm);
Result := 0;
Exit;
end;
//large or medium?
if NativeUInt(pm.OwnerBlock) and 2 = 0 then
Result := FMediumMemManager.FreeMem(aMemory)
else
Result := FLargeMemManager.FreeMemWithHeader(aMemory)
end
else
//small mem
begin
ot := pm.OwnerBlock.OwnerManager;
if ot = @FSmallMemManager then
Result := FSmallMemManager.FreeMem(aMemory)
else
begin
FreeMemOfOtherThread(pm);
Result := 0;
end;
end;
{$IFDEF SCALEMM_DEBUG}
CheckMem(nil);
{$ENDIF}
end;
function TThreadMemManager.FreeMemFromOtherThread(
aMemory: PBaseMemHeader): NativeInt;
var
ot: PBaseSizeManager;
op: PThreadMemManager;
p: Pointer;
begin
//check double free
if (aMemory.Size and 1 <> 0) then
Error(reInvalidPtr);
//convert to "client" pointer again to be able to use the normal functions
p := Pointer(NativeUInt(aMemory) + SizeOf(TBaseMemHeader));
//large or medium?
if NativeUInt(aMemory.OwnerBlock) and 3 <> 0 then
begin
op := PThreadMemManager( NativeUInt(aMemory.OwnerBlock) and -4);
//check owner (can be changed in the meantime!)
if op <> @Self then
begin
FreeMemOfOtherThread(aMemory);
Result := 0;
Exit;
end;
//large or medium?
if NativeUInt(aMemory.OwnerBlock) and 2 = 0 then
Result := FMediumMemManager.FreeMem(p)
else
Result := FLargeMemManager.FreeMemWithHeader(p)
end
else
begin
ot := aMemory.OwnerBlock.OwnerManager;
//check owner (can be changed in the meantime!)
if ot = @FSmallMemManager then
Result := FSmallMemManager.FreeMem(p)
else
begin
FreeMemOfOtherThread(aMemory);
Result := 0;
end;
end;
{$ifdef SCALEMM_DEBUG}
if not Self.FThreadTerminated then
Self.CheckMem(nil);
{$ENDIF}
end;
{
procedure TThreadMemManager.AddFreeMemToOwnerThread(aFirstMem,
aLastMem: PBaseFreeMemHeader);
begin
//LOCK
Lock;
//put new mem to front of linked list
aLastMem.NextThreadFree := FOtherThreadFreedMemory;
FOtherThreadFreedMemory := aFirstMem;
//UNLOCK
Unlock;
end;
}
function TThreadMemManager.GetMem(aSize: NativeInt): Pointer;
begin
if FOtherThreadFreedMemory <> nil then
ProcessFreedMemFromOtherThreads(True);
if aSize <= C_MAX_SMALLMEM_SIZE then //-1 till 2048
begin
if aSize > 0 then
Result := FSmallMemManager.GetMem(aSize)
else
begin
Result := nil;
Exit;
end
end
else if aSize <= C_MAX_MEDIUMMEM_SIZE - SizeOf(TMediumHeader) then //till 1Mb
Result := FMediumMemManager.GetMem(aSize)
else
begin
Result := FLargeMemManager.GetMemWithHeader(aSize);
end;
Assert( NativeUInt(Result) AND 3 = 0);
{$IFDEF Align8Bytes}
Assert( NativeUInt(Result) AND 7 = 0);
{$ENDIF}
{$IFDEF Align16Bytes}
Assert( NativeUInt(Result) AND 15 = 0);
{$ENDIF}
{$IFDEF SCALEMM_DEBUG}
CheckMem(nil);
{$ENDIF}
end;
procedure TThreadMemManager.Init;
begin
FThreadId := GetCurrentThreadId;
FSmallMemManager.Init;
FSmallMemManager.OwnerThread := @Self;
FMediumMemManager.Init;
FMediumMemManager.OwnerThread := @Self;
FLargeMemManager.Init;
FLargeMemManager.OwnerThread := @Self;
{$IFDEF SCALEMM_DEBUG}
CheckMem(nil);
{$ENDIF}
end;
function TThreadMemManager.IsMemoryFromOtherThreadsPresent: Boolean;
begin
Result := (FOtherThreadFreedMemory <> nil) or
FSmallMemManager.IsMemoryFromOtherThreadsPresent;
end;
{$if CompilerVersion >= 23} //Delphi XE2
function Scale_ReallocMem(aMemory: Pointer; aSize: NativeInt): Pointer;
{$else}
function Scale_ReallocMem(aMemory: Pointer; aSize: Integer): Pointer;
{$ifend}
var
pm: PBaseMemHeader;
iSize: NativeUInt;
begin
// ReAlloc can be misued as GetMem or FreeMem (documented in delphi help) so check what the user wants
// Normal realloc of exisiting data?
if (aMemory <> nil) and (aSize > 0) then
begin
//general resize: if size within 1/4 we do nothing (also possible in other thread!)
//iSize := NativeUInt(Pointer(NativeUInt(aMemory) - SizeOf(TBaseMemHeader))^);
pm := PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader));
iSize := pm.Size;
//downsize...
if (NativeUInt(aSize) <= iSize) then
begin
Result := aMemory;
if iSize <= C_MAX_SMALLMEM_SIZE then //small mem?
begin
//within 1/4?
if (NativeUInt(aSize) + 32 > iSize shr 2) then
Exit;
end
else
begin
//medium + large mem has included their header size in the size too
if iSize <= C_MAX_MEDIUMMEM_SIZE then //medium mem?
begin
Assert( NativeUInt(PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader)).OwnerBlock) and 2 = 0 ); //must be marked as medium!
//within 1/2?
if (NativeUInt(aSize) + SizeOf(TMediumHeader) <= iSize) then
begin
if (NativeUInt(aSize) > iSize shr 1) then
Exit
end
else
begin
Result := GetThreadMemManager.ReallocMem(aMemory, aSize + (aSize shr 3)); //add extra size (12,5%)
Exit;
end;
end
else //large mem
begin
Assert( NativeUInt(PBaseMemHeader(NativeUInt(aMemory) - SizeOf(TBaseMemHeader)).OwnerBlock) and 2 <> 0); //must marked as large!
//within 1/2?
if (NativeUInt(aSize) + SizeOf(TLargeHeader) {+ SizeOf(TLargeBlockMemory)} <= iSize) then
begin
if (NativeUInt(aSize) > iSize shr 1) then
Exit
end
else
begin
Result := GetThreadMemManager.ReallocMem(aMemory, aSize + (aSize shr 4)); //add extra size (1/16, 6,25%) for large mem
Exit;
end;
end;
end;
//too much downsize: realloc anyway
Result := GetThreadMemManager.GetMem(aSize);
Move(aMemory^, Result^, aSize); // copy (use smaller new size)
Scale_FreeMem(aMemory); //free mem (possible from other thread!)
Exit;
end;
//normal realloc
if iSize <= C_MAX_MEDIUMMEM_SIZE then //small or medium mem?
Result := GetThreadMemManager.ReallocMem(aMemory, aSize + (aSize shr 2) ) //add extra size (1/4, 25%)
else
Result := GetThreadMemManager.ReallocMem(aMemory, aSize + (aSize shr 4) ) //add extra size (1/16, 6,25%) for large mem
end
else
begin
if (aMemory = nil) and (aSize > 0) then
// GetMem disguised as ReAlloc
Result := Scale_GetMem(aSize)
else
begin
// FreeMem disguised as ReAlloc
Result := nil;
Scale_FreeMem(aMemory);
end;
end;
end;
{$if CompilerVersion >= 23} //Delphi XE2
function Scale_GetMem(aSize: NativeInt): Pointer;
{$else}
function Scale_GetMem(aSize: Integer): Pointer;
{$ifend}
{$IFDEF HASINLINE}
begin
Result := GetThreadMemManager.GetMem(aSize);
end;
{$ELSE}
{$IFDEF PURE_PASCAL}
begin
Result := GetThreadMemManager.GetMem(aSize);
end;
{$ELSE}
asm
{$IFDEF INLINEGOWN}
mov edx,eax
mov eax,GOwnTlsOffset
mov ecx,fs:[$00000018]
mov eax,[ecx+eax] // fixed offset, calculated only once
or eax,eax
jnz TThreadMemManager.GetMem
push edx
call CreateMemoryManager
pop edx
jmp TThreadMemManager.GetMem
{$ELSE}
push eax
call GetThreadMemManager
pop edx
jmp TThreadMemManager.GetMem
{$endif}
end;
{$ENDIF}
{$ENDIF}
{$if CompilerVersion >= 23} //Delphi XE2
function Scale_AllocMem(aSize: NativeInt): Pointer;
{$else}
function Scale_AllocMem(aSize: Cardinal): Pointer;
{$ifend}
begin
Result := Scale_GetMem(aSize);
fillchar(Result^, aSize, 0); // AllocMem() = GetMem()+ZeroMemory()
end;
function Scale_FreeMem(aMemory: Pointer): Integer;
{$IFDEF HASINLINE}
begin
Result := GetThreadMemManager.FreeMem(aMemory);
end;
{$ELSE}
{$IFDEF PURE_PASCAL}
begin
Result := GetThreadMemManager.FreeMem(aMemory);
end;
{$ELSE}
asm
{$IFDEF INLINEGOWN}
mov edx,eax
mov eax,GOwnTlsOffset
mov ecx,fs:[$00000018]
mov eax,[ecx+eax] // fixed offset, calculated only once
or eax,eax
jnz TThreadMemManager.FreeMem
push edx
call CreateMemoryManager
pop edx
jmp TThreadMemManager.FreeMem
{$ELSE}
push eax
call GetThreadMemManager
pop edx
jmp TThreadMemManager.FreeMem
{$endif}
end;
{$ENDIF}
{$ENDIF}
procedure Scale_CheckMem(aMemory: Pointer);
begin
GetThreadMemManager.CheckMem(aMemory);
end;
{$ifdef USEMEMMANAGEREX}
function Scale_RegisterMemoryLeak(P: Pointer): Boolean;
begin
{ TODO : implement memory leak checking }
// Result := OldMM.RegisterExpectedMemoryLeak(p);
Result := True;
end;
function Scale_UnregisterMemoryLeak(P: Pointer): Boolean;
begin
// Result := OldMM.UnregisterExpectedMemoryLeak(p);
Result := True;
end;
{$endif}
type
TEndThread = procedure(ExitCode: Integer);
PEndThread = ^TEndThread;
var
// OldEndThread: TEndThread;
NewEndThreadProc: PEndThread;
procedure NewEndThread(ExitCode: Integer); //register; // ensure that calling convension matches EndThread
begin
// free all thread mem
GlobalManager.FreeThreadManager( GetThreadMemManager );
// OldEndThread(ExitCode); todo: make trampoline with original begin etc
// code of original EndThread;
ExitThread(ExitCode);
end;
type
PJump = ^TJump;
TJump = packed record
OpCode : Byte;
Distance: Integer;
end;
procedure FastcodeAddressPatch(const ASource, ADestination: Pointer);
const
Size: NativeInt = SizeOf(TJump);