forked from UCBoulder/GEM_Intel
-
Notifications
You must be signed in to change notification settings - Fork 0
/
gem_com.F90
335 lines (272 loc) · 14.2 KB
/
gem_com.F90
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
!$ include "mkl_omp_offload.f90"
module gem_com
!common data used for gem
#if defined(MKL_ILP64)
!$ use onemkl_lapack_omp_offload_ilp64 ! 64-bit
#else
!$ use onemkl_lapack_omp_offload_lp64 ! 32-bit
#endif
use mpi
use, intrinsic :: iso_c_binding
use gem_pputil
implicit none
INTERFACE
real function revers(num,n)
end function revers
real function ran2(i)
end function ran2
real function en3(s)
real :: s
end function en3
END INTERFACE
integer :: imx,jmx,kmx,mmx,mmxe,nmx,nsmx,nsubd=8,&
ntube,nxpp,ngdx=5,nb=6, &
negrd=16,nlgrd=16
integer :: gemout=777
character(len=70) outname
REAL :: endtm,begtm,pstm
REAL :: starttm,lasttm,tottm
real :: start_ppush_tm, end_ppush_tm, tot_ppush_tm = 0, start_cpush_tm, end_cpush_tm, tot_cpush_tm = 0, start_pint_tm, end_pint_tm, tot_pint_tm = 0, start_cint_tm, end_cint_tm, tot_cint_tm = 0, start_grid_tm, end_grid_tm, tot_grid_tm = 0
real :: start_jie_tm, end_jie_tm, tot_jie_tm = 0, start_den0_tm, end_den0_tm, tot_den0_tm = 0, start_setw_tm, end_setw_tm, tot_setw_tm = 0, start_weatxeps_tm, end_weatxeps_tm, tot_weatxeps_tm = 0, start_wiatxeps_tm, end_wiatxeps_tm, tot_wiatxeps_tm = 0, start_pbi_tm, end_pbi_tm, tot_pbi_tm = 0, start_pbe_tm, end_pbe_tm, tot_pbe_tm = 0, start_jpar0_tm, end_jpar0_tm, tot_jpar0_tm = 0
real :: start_init_pmove_tm, end_init_pmove_tm, tot_init_pmove_tm = 0
real :: start_pmove_tm, end_pmove_tm, tot_pmove_tm = 0
real :: start_poisson_tm, end_poisson_tm, tot_poisson_tm = 0
real :: start_ampere_tm, end_ampere_tm, tot_ampere_tm = 0
real :: start_init_lap_tm, end_init_lap_tm, tot_init_lap_tm = 0
real :: aux1(50000),aux2(20000)
real,dimension(:),allocatable :: workx,worky,workz,xsinin,xsinout
complex,dimension(:),allocatable :: tmpx,xin,xout,yin,yout,zin,zout
complex,dimension(:),allocatable :: tmpy
complex,dimension(:),allocatable :: tmpz
complex,dimension(:,:),allocatable :: tmpxyin,tmpxyout
type(C_PTR) :: planx,iplanx,plany,iplany,planz,iplanz,plansinx
integer :: icgp,jcgp,cgpfacx,cgpfacy,nxsrc,nesrc
integer :: mme,mmb
REAL, dimension(:,:),allocatable :: rwx,rwy
INTEGER,dimension(:),allocatable :: mm,tmm,lr
integer :: micell,mecell !jycheng
integer :: nonlin1,nonlin2 !jycheng
REAL,dimension(:),allocatable :: tets,mims,q
REAL,dimension(:),allocatable :: kapn, kapt
INTEGER :: timestep,im,jm,km,mykm,iseed,nrst,nfreq,isft,mynf,ifskp,iphbf,iapbf,idpbf
real,dimension(:),allocatable :: time
REAL :: dx,dy,dxcgp,dycgp,dz,dxsrc,desrc,ecutsrc,pi,pi2,dt,dte,totvol,n0,n0e,tcurr,rmpp,rmaa,eprs
REAL :: lx,ly,lz,xshape,yshape,zshape,pzcrit(5),pzcrite,encrit,tot_field_e,tot_joule,tot_joule1
INTEGER :: nm,nsm,kcnt,jcnt,ncurr,llk,mlk,onemd,iflr,iorb
integer :: izonal,adiabatic_electron,ineq0,iflut,nlow,ntor0,mstart,iexb
REAL :: cut,amp,tor,amie,isg,rneu,rneui,emass,qel,mbeam,qbeam,teth,vexbsw,vparsw,gammah,gamgtc,gamtoy,ghzon,gamgyro
REAL :: c4,fradi,kxcut,kycut,bcut,wecut,ftrap,adwn,adwe,adwp,frmax
INTEGER :: iput,iget,igetmx,idg,kzlook,ision,isiap,peritr,iadi,ipred,icorr,jpred,jcorr
REAL,DIMENSION(:,:),allocatable :: yyamp,yyre,yyim
complex,dimension(:,:),allocatable :: camp,campf
REAL :: br0,lr0,qp,e0,vcut,vpp,vt0,yd0,cv,vsphere
integer :: nonlin(5),nonline,ipara,isuni,isunie,ifluid,nopz,nopi(5),nowi(5),noen,nowe,novpar,isonew
integer :: ipbm,ias
complex :: IU
real,dimension(:),allocatable :: coefx,coefy,coefz
complex,dimension(1:8) :: apk,ptk,dpdtk
integer,dimension(1:8) :: lapa,mapa,napa
real :: mrtio(0:1),aven,avptch
integer :: icrs_sec,ipg,isphi
integer,dimension(0:255) :: isgnft,jft
REAL,DIMENSION(:,:,:,:),allocatable :: den
REAL,DIMENSION(:,:,:,:),allocatable :: dnidt,jpar,jpex,jpey,dti
REAL,DIMENSION(:,:,:),allocatable :: rho,jion,jionx,jiony
real,dimension(:,:,:),allocatable :: phi
real,dimension(:,:,:),allocatable :: drhodt,dnedt,dphidt,drhoidt
REAL,DIMENSION(:,:,:),allocatable :: ex
REAL,DIMENSION(:,:,:),allocatable :: ey
REAL,DIMENSION(:,:,:),allocatable :: ez
REAL,DIMENSION(:,:,:),allocatable :: dpdz,dadz
REAL,DIMENSION(:,:,:),allocatable :: delbx,delby
REAL,DIMENSION(:),allocatable :: xg,yg,zg
real,dimension(:,:,:),allocatable :: apar,dene
real,dimension(:,:,:),allocatable :: upar,upart,delte
real,dimension(:,:,:),allocatable :: upex,upey,upa0,den0,upazd,upa00,upa0t,den0apa
real,dimension(:,:),allocatable :: cfx,cfy,jac,bmag,bdgxcgy,bdgrzn,ggxdgy,ggy2,ggx,gthf
real,dimension(:),allocatable :: gn0e,gt0e,gt0i,avap,dtez,gsf
real,dimension(:,:),allocatable :: gn0s,gt0s,dtiz
!mv/pt variables
real,dimension(:,:,:),allocatable :: apars,aparss,aparh,lapapar,delbxh,delbyh,dahdz,aparh_save
! particle array declarations
REAL,DIMENSION(:,:),allocatable :: mu,xii,pzi,eki,z0i,u0i
REAL,DIMENSION(:,:),allocatable :: x2,y2,z2,u2
REAL,DIMENSION(:,:),allocatable :: x3,y3,z3,u3
REAL,DIMENSION(:,:),allocatable :: w2,w3
REAL,DIMENSION(:),allocatable :: mue,xie,pze,eke,z0e,u0e
REAL,DIMENSION(:),allocatable :: x2e,y2e,z2e,u2e,mue2
REAL,DIMENSION(:),allocatable :: x3e,y3e,z3e,u3e,mue3
REAL,DIMENSION(:),allocatable :: w2e,w3e
real,dimension(:),allocatable :: ipass, index
REAL,DIMENSION(:),allocatable :: w000,w001,w010,w011,w100,w101,w110,w111
! NCG variables
REAL,DIMENSION(:),allocatable :: xtmp,ytmp,utmp
REAL,DIMENSION(:,:,:,:,:),allocatable :: thisg,totg,thish,toth1,toth2
! source variables
REAL,DIMENSION(:,:),allocatable :: avwexeps,gmrkre,avwexez !e 2D
REAL,DIMENSION(:),allocatable :: fesrc,dnesrc,dnesrcz !e 1D
REAL,DIMENSION(:,:,:),allocatable :: avwixeps,gmrkr,avwixez !i 3D
REAL,DIMENSION(:,:),allocatable :: fisrc,dnisrc,dnisrcz !i 2D
integer,DIMENSION(:,:),allocatable :: numxeps
! Various diagnostic arrays and scalars
! plotting constants
INTEGER :: nplot,xnplt,imovie=1000000000,nzcrt,npze,npzi,npzc,npzb,nzsrc,nzgyro
REAL :: contu,wmax
! energy diagnostic arrays
REAL,DIMENSION(:,:),allocatable :: ke
REAL,DIMENSION(:),allocatable :: fe,te
REAL,DIMENSION(:),allocatable :: rmsphi,rmsapa,avewe
REAL,DIMENSION(:,:),allocatable :: nos,avewi
! flux diagnostics
REAL,DIMENSION(:),allocatable :: vol
REAL,DIMENSION(:,:),allocatable :: efle_es,efle_em,pfle_es,pfle_em
REAL,DIMENSION(:,:,:),allocatable :: pfl_es,pfl_em,efl_es,efl_em
REAL,DIMENSION(:,:),allocatable :: chii, chie, ddi
REAL,DIMENSION(:),allocatable :: achii, achie, addi
real,dimension(:),allocatable :: mdhis,mdhisa,mdhisb,mdhisc,mdhisd
complex,dimension(:,:),allocatable :: aparhis,phihis
! kr, ktheta spectrum plots
REAL,DIMENSION(:,:),allocatable :: phik
! weighty variables
INTEGER,dimension(:),allocatable :: deljp,deljm
INTEGER,dimension(:,:),allocatable :: jpl
INTEGER,dimension(:,:),allocatable :: jpn
INTEGER,dimension(:,:),allocatable :: jmi
INTEGER,dimension(:,:),allocatable :: jmn
REAL,DIMENSION(:),allocatable :: weightp,weightm
REAL,DIMENSION(:),allocatable :: weightpn,weightmn
!blending variable
complex,dimension(:,:,:,:),allocatable :: pol,pmtrx,pmtrxi
complex,dimension(:,:),allocatable :: pfac
complex,dimension(:,:,:,:),allocatable :: mxg,mxa,mxd
integer,dimension(:,:,:,:),allocatable :: ipivg,ipiva,ipivd
! MPI variables
! include '/usr/include/mpif.h'
integer,parameter :: Master=0
integer :: numprocs,n_omp
INTEGER :: MyId,Last,cnt,ierr
INTEGER :: GRID_COMM,TUBE_COMM
INTEGER :: GCLR,TCLR,GLST,TLST
INTEGER :: stat(MPI_STATUS_SIZE)
INTEGER :: lngbr,rngbr,idprv,idnxt
character(len=*) directory
parameter(directory='./dump/')
character(len=*) outdir
parameter(outdir='./out/')
!real :: ran2,revers
!integer :: mod
!real :: amod
save
contains
subroutine new_gem_com()
nxpp = imx !/ntube
allocate(workx(4*imx),worky(4*jmx),workz(4*kmx),xsinin(imx),xsinout(imx))
allocate(tmpx(0:imx-1),xin(imx),xout(imx),yin(jmx),yout(jmx),zin(kmx),zout(kmx))
allocate(tmpy(0:jmx-1))
allocate(tmpz(0:kmx-1))
allocate(tmpxyin(1:imx,1:jmx),tmpxyout(1:imx,1:jmx))
allocate(rwx(5,4),rwy(5,4))
allocate(mm(5),tmm(5),lr(5))
allocate(tets(5),mims(5),q(5))
allocate(kapn(5),kapt(5))
allocate(time(0:nmx))
allocate(yyamp(0:jmx,0:4),yyre(0:jmx,0:4),yyim(0:jmx,0:4),camp(0:6,0:50000),campf(0:6,0:nfreq-1))
allocate(aparhis(0:6,0:jcnt-1),phihis(0:6,0:jcnt-1))
allocate(mdhis(0:100),mdhisa(0:100),mdhisb(0:100))
allocate(mdhisc(0:100),mdhisd(0:100))
allocate(coefx(100+8*imx),coefy(100+8*jmx),coefz(100+8*kmx))
ALLOCATE( den(nsmx,0:nxpp,0:jmx,0:1),dti(nsmx,0:nxpp,0:jmx,0:1), &
delte(0:nxpp,0:jmx,0:1))
ALLOCATE( rho(0:nxpp,0:jmx,0:1),drhoidt(0:nxpp,0:jmx,0:1), &
jion(0:nxpp,0:jmx,0:1),jionx(0:nxpp,0:jmx,0:1), &
jiony(0:nxpp,0:jmx,0:1))
allocate( phi(0:nxpp,0:jmx,0:1))
allocate( drhodt(0:nxpp,0:jmx,0:1),dnedt(0:nxpp,0:jmx,0:1))
allocate( dnidt(nsmx,0:nxpp,0:jmx,0:1),jpar(nsmx,0:nxpp,0:jmx,0:1), &
jpex(nsmx,0:nxpp,0:jmx,0:1),jpey(nsmx,0:nxpp,0:jmx,0:1))
allocate( dphidt(0:nxpp,0:jmx,0:1))
ALLOCATE( ex(0:nxpp,0:jmx,0:1))
ALLOCATE( ey(0:nxpp,0:jmx,0:1))
ALLOCATE( ez(0:nxpp,0:jmx,0:1))
ALLOCATE( dpdz(0:nxpp,0:jmx,0:1),dadz(0:nxpp,0:jmx,0:1))
ALLOCATE( delbx(0:nxpp,0:jmx,0:1),delby(0:nxpp,0:jmx,0:1))
ALLOCATE( xg(0:nxpp),yg(0:jmx),zg(0:1))
allocate( apar(0:nxpp,0:jmx,0:1),dene(0:nxpp,0:jmx,0:1))
allocate( upar(0:nxpp,0:jmx,0:1),upart(0:nxpp,0:jmx,0:1),upex(0:nxpp,0:jmx,0:1), &
upey(0:nxpp,0:jmx,0:1),upa0(0:nxpp,0:jmx,0:1), &
den0(0:nxpp,0:jmx,0:1),upazd(0:nxpp,0:jmx,0:1),&
upa00(0:nxpp,0:jmx,0:1),upa0t(0:nxpp,0:jmx,0:1),den0apa(0:nxpp,0:jmx,0:1))
allocate( cfx(0:nxpp,0:1),cfy(0:nxpp,0:1),jac(0:nxpp,0:1))
allocate( bmag(0:nxpp,0:1),bdgxcgy(0:nxpp,0:1),bdgrzn(0:nxpp,0:1),gthf(0:nxpp,0:1),gsf(0:nxpp))
allocate( ggxdgy(0:nxpp,0:1),ggy2(0:nxpp,0:1),ggx(0:nxpp,0:1))
allocate (gn0e(0:nxpp),gt0e(0:nxpp),gt0i(0:nxpp),avap(0:nxpp),dtez(0:imx))
allocate (gn0s(1:5,0:nxpp),gt0s(1:5,0:nxpp),dtiz(1:5,0:imx))
if(ipbm==1)then
allocate(apars(0:nxpp,0:jmx,0:1),aparss(0:nxpp,0:jmx,0:1),aparh(0:nxpp,0:jmx,0:1),lapapar(0:nxpp,0:jmx,0:1), &
delbxh(0:nxpp,0:jmx,0:1),delbyh(0:nxpp,0:jmx,0:1),dahdz(0:nxpp,0:jmx,0:1),aparh_save(0:nxpp,0:jmx,0:1))
end if
! particle array declarations
!allocate( mu(nsmx,1:mmx),xii(nsmx,1:mmx),pzi(nsmx,1:mmx), &
! eki(nsmx,1:mmx),z0i(nsmx,1:mmx),u0i(nsmx,1:mmx))
!allocate( x2(nsmx,1:mmx),y2(nsmx,1:mmx),z2(nsmx,1:mmx),u2(nsmx,1:mmx))
!allocate( x3(nsmx,1:mmx),y3(nsmx,1:mmx),z3(nsmx,1:mmx),u3(nsmx,1:mmx))
!allocate( w2(nsmx,1:mmx),w3(nsmx,1:mmx))
allocate( mu(1:mmx,nsmx),xii(1:mmx,nsmx),pzi(1:mmx,nsmx), &
eki(1:mmx,nsmx),z0i(1:mmx,nsmx),u0i(1:mmx,nsmx))
allocate( x2(1:mmx,nsmx),y2(1:mmx,nsmx),z2(1:mmx,nsmx),u2(1:mmx,nsmx))
allocate( x3(1:mmx,nsmx),y3(1:mmx,nsmx),z3(1:mmx,nsmx),u3(1:mmx,nsmx))
allocate( w2(1:mmx,nsmx),w3(1:mmx,nsmx))
!$acc enter data create(mu,xii,pzi,eki,z0i,u0i,x2,y2,z2,u2,x3,y3,z3,u3,w2,w3)
!$omp target enter data map(alloc:mu,xii,pzi,eki,z0i,u0i,x2,y2,z2,u2,&
!$omp x3,y3,z3,u3,w2,w3)
allocate( mue(1:mmxe),xie(1:mmxe),pze(1:mmxe),eke(1:mmxe),z0e(1:mmxe),u0e(1:mmxe))
allocate( x2e(1:mmxe),y2e(1:mmxe),z2e(1:mmxe),u2e(1:mmxe),mue2(1:mmxe))
allocate( x3e(1:mmxe),y3e(1:mmxe),z3e(1:mmxe),u3e(1:mmxe),mue3(1:mmxe))
allocate( w2e(1:mmxe),w3e(1:mmxe))
allocate( ipass(1:mmxe), index(1:mmxe))
allocate(w000(1:mmxe),w001(1:mmxe),w010(1:mmxe),w011(1:mmxe),&
w100(1:mmxe),w101(1:mmxe),w110(1:mmxe),w111(1:mmxe))
!$acc enter data create(mue,xie,pze,eke,z0e,u0e,x2e,y2e,z2e,u2e,mue2,x3e,y3e,z3e,u3e,mue3)
!$omp target enter data map(alloc:mue,xie,pze,eke,z0e,u0e,x2e,y2e,z2e,&
!$omp u2e,mue2,x3e,y3e,z3e,u3e,mue3)
!$acc enter data create(w2e,w3e,ipass,index,w000,w001,w010,w011,w100,w101,w110,w111)
!$omp target enter data map(alloc:w2e,w3e,ipass,index,w000,w001,w010,&
!$omp w011,w100,w101,w110,w111)
!7/24/2019 YCHEN global 5D arrays for nonlinear CGP.
! allocate( xtmp(1:mmxe),ytmp(1:mmxe),utmp(1:mmxe))
! allocate( thisg(0:icgp,0:jcgp,0:1,0:negrd,0:nlgrd),totg(0:icgp,0:jcgp,0:1,0:negrd,0:nlgrd),thish(0:icgp,0:jcgp,0:1,0:negrd,0:nlgrd), &
! toth1(0:icgp,0:jcgp,0:1,0:negrd,0:nlgrd),toth2(0:icgp,0:jcgp,0:1,0:negrd,0:nlgrd))
allocate(avwexeps(0:nxsrc-1,0:nesrc-1),fesrc(0:nxsrc-1),dnesrc(0:nxsrc-1),gmrkr(nsmx,0:nxsrc-1,0:nesrc-1),gmrkre(0:nxsrc-1,0:nesrc-1),avwexez(0:nxsrc-1,0:nesrc-1))
allocate(avwixeps(nsmx,0:nxsrc-1,0:nesrc-1),fisrc(nsmx,0:nxsrc-1),dnisrc(nsmx,0:nxsrc-1),avwixez(nsmx,0:nxsrc-1,0:nesrc-1))
allocate(dnisrcz(nsmx,0:nxsrc-1),dnesrcz(0:nxsrc-1))
! Various diagnostic arrays and scalars
! plotting constants
ALLOCATE( ke(nsmx,0:nmx),fe(0:nmx),te(0:nmx))
ALLOCATE( rmsphi(0:nmx),rmsapa(0:nmx),avewi(1:3,0:nmx),avewe(0:nmx))
ALLOCATE( nos(nsmx,0:nmx))
! flux diagnostics
ALLOCATE( vol(1:nsubd),efle_es(1:nsubd,0:nmx),pfle_es(1:nsubd,0:nmx), &
pfl_es(1:3,1:nsubd,0:nmx),efl_es(1:3,1:nsubd,0:nmx), &
pfle_em(1:nsubd,0:nmx),efle_em(1:nsubd,0:nmx), &
pfl_em(1:3,1:nsubd,0:nmx),efl_em(1:3,1:nsubd,0:nmx))
ALLOCATE( chii(1:nsubd,0:nmx),chie(1:nsubd,0:nmx),ddi(1:nsubd,0:nmx), &
achii(1:nsubd),achie(1:nsubd),addi(1:nsubd))
! kr, ktheta spectrum plots
ALLOCATE( phik(imx,jmx))
! weighty variables
ALLOCATE( deljp(0:nxpp),deljm(0:nxpp))
ALLOCATE( jpl(0:nxpp,0:jmx))
ALLOCATE( jpn(0:nxpp,0:jmx))
ALLOCATE( jmi(0:nxpp,0:jmx))
ALLOCATE( jmn(0:nxpp,0:jmx))
ALLOCATE( weightp(0:nxpp),weightm(0:nxpp))
ALLOCATE( weightpn(0:nxpp),weightmn(0:nxpp))
!Blending variable
ALLOCATE(pol(1:nb,0:imx-1,0:jmx-1,0:kmx),pfac(0:imx-1,0:jmx-1), &
pmtrx(0:imx-1,0:jmx-1,1:nb,1:nb), &
pmtrxi(0:imx-1,0:jmx-1,1:nb,1:nb))
allocate(mxg(imx-1,imx-1,0:jcnt-1,0:1),mxa(imx-1,imx-1,0:jcnt-1,0:1),mxd(imx-1,imx-1,0:jcnt-1,0:1), &
ipivg(imx-1,imx-1,0:jcnt-1,0:1),ipiva(imx-1,imx-1,0:jcnt-1,0:1),ipivd(imx-1,imx-1,0:jcnt-1,0:1))
end subroutine new_gem_com
end module gem_com
! Code was translated using: /nfs/site/home/tkloeffe/TCE/intel-application-migration-tool-for-openacc-to-openmp/src/intel-application-migration-tool-for-openacc-to-openmp gem_com.F90