From 43199aa28bb41a67e16308f356f5fb49f2e3cde5 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 19 Jul 2024 12:42:33 +0100 Subject: [PATCH 01/15] Fix time lag issue for inversion - t passed to cost_fn(t) was out of sync by one timestep so the optimal result was not correct for channel inversion - Now need to pass the cost function through the export_func rather than through update_forcings - Typo fixed in Tohoku Makefile as well --- examples/channel_inversion/inverse_problem.py | 2 +- examples/tohoku_inversion/Makefile | 2 +- examples/tohoku_inversion/inverse_problem.py | 2 +- thetis/inversion_tools.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/channel_inversion/inverse_problem.py b/examples/channel_inversion/inverse_problem.py index 5778b39de..e480512eb 100644 --- a/examples/channel_inversion/inverse_problem.py +++ b/examples/channel_inversion/inverse_problem.py @@ -113,7 +113,7 @@ cost_function = inv_manager.get_cost_function(solver_obj) # Solve and setup reduced functional -solver_obj.iterate(update_forcings=cost_function) +solver_obj.iterate(export_func=cost_function) inv_manager.stop_annotating() # Run inversion diff --git a/examples/tohoku_inversion/Makefile b/examples/tohoku_inversion/Makefile index 786348675..03a428dce 100644 --- a/examples/tohoku_inversion/Makefile +++ b/examples/tohoku_inversion/Makefile @@ -9,7 +9,7 @@ invert: plot: python3 plot_elevation_progress.py - python3 plot_elevation_optimised.py + python3 plot_elevation_optimized.py python3 plot_convergence.py clean: diff --git a/examples/tohoku_inversion/inverse_problem.py b/examples/tohoku_inversion/inverse_problem.py index 365a5ade4..c72858e8c 100644 --- a/examples/tohoku_inversion/inverse_problem.py +++ b/examples/tohoku_inversion/inverse_problem.py @@ -101,7 +101,7 @@ cost_function = inv_manager.get_cost_function(solver_obj, weight_by_variance=True) # Solve and setup the reduced functional -solver_obj.iterate(update_forcings=cost_function) +solver_obj.iterate(export_func=cost_function) inv_manager.stop_annotating() # Run inversion diff --git a/thetis/inversion_tools.py b/thetis/inversion_tools.py index 36d762d0a..232f0c5e2 100644 --- a/thetis/inversion_tools.py +++ b/thetis/inversion_tools.py @@ -252,7 +252,8 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): var.dat.data[i] = numpy.var(self.sta_manager.observation_values[j]) self.sta_manager.station_weight_0d.interpolate(1/var) - def cost_fn(t): + def cost_fn(): + t = solver_obj.simulation_time misfit = self.sta_manager.eval_cost_function(t) self.J_misfit += misfit self.J += misfit From 8f2b7d362152f2d1b9a67449890d09054b549a76 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 19 Jul 2024 15:53:11 +0100 Subject: [PATCH 02/15] Add headland inversion example - Details are in the README - inversion of Manning field using different field representations: - Constant - Region-based - Independent Points Scheme - Free specification at all points in the domain (Hessian regularised) --- examples/headland_inversion/Makefile | 17 + examples/headland_inversion/README.md | 199 + examples/headland_inversion/headland.geo | 51 + examples/headland_inversion/headland.msh | 4042 +++++++++++++++++ .../images/seabed_classification.png | Bin 0 -> 45932 bytes .../headland_inversion/inverse_problem.py | 283 ++ .../headland_inversion/inversion_tools_vel.py | 797 ++++ examples/headland_inversion/model_config.py | 186 + .../plot_velocity_progress.py | 77 + test/examples/test_examples.py | 2 + test_adjoint/examples/test_examples.py | 3 +- thetis/utility.py | 9 + 12 files changed, 5665 insertions(+), 1 deletion(-) create mode 100644 examples/headland_inversion/Makefile create mode 100644 examples/headland_inversion/README.md create mode 100644 examples/headland_inversion/headland.geo create mode 100644 examples/headland_inversion/headland.msh create mode 100644 examples/headland_inversion/images/seabed_classification.png create mode 100644 examples/headland_inversion/inverse_problem.py create mode 100644 examples/headland_inversion/inversion_tools_vel.py create mode 100644 examples/headland_inversion/model_config.py create mode 100644 examples/headland_inversion/plot_velocity_progress.py diff --git a/examples/headland_inversion/Makefile b/examples/headland_inversion/Makefile new file mode 100644 index 000000000..1a5c6ad5a --- /dev/null +++ b/examples/headland_inversion/Makefile @@ -0,0 +1,17 @@ +all: invert plot + +CASE ?= IndependentPointsScheme # Constant Regions NodalFreedom +CONTROLS = Manning # This option is not used in this example - Manning only +STATIONS = stationA stationB stationC stationD stationE + +invert: + python3 inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test; \ + +plot: + for station in $(STATIONS); do \ + python3 plot_velocity_progress.py -s $$station --case $(CASE); \ + done; \ + +clean: + rm -rf __pycache__ + rm -rf *.png diff --git a/examples/headland_inversion/README.md b/examples/headland_inversion/README.md new file mode 100644 index 000000000..0f2a1e6bb --- /dev/null +++ b/examples/headland_inversion/README.md @@ -0,0 +1,199 @@ +# Headland Channel - Friction Field Calibration + +## Forward run + +The forward run is not included here, instead, only the time series `.hdf5` files at each station are provided. + +The forward run used to generate this uses the same model configuration as provided by `model_config.py` which +configures the `solver object`, but with a friction field based on a sea bed particle sizes as shown below: + +![Sea Bed Particle Sizes](images/seabed_classification.png) + +The idealised headland is 20km long and 6km wide, with a coastline depth of 3m and a main channel depth of 40m. Both +boundaries are undefined and are thus land boundaries. The boundaries are forced by a sinusoidal elevation function, +emulating a single tidal signal. A viscosity sponge is used at the left hand boundary to provide some model stability. + +## Inversion run + +### Full Nodal Flexibility + +```sh +source ~/firedrake/bin/activate +make invert CASE=NodalFreedom +``` + +The inversion problem is currently run from a `Makefile`. User arguments are specified here i.e. fields to optimise and +then the Makefile runs the scripts with the inputs provided. + +The solver object is set up using `construct_solver` and then initial values for each field (in this case we only +optimise for bed friction) are specified. The station manager, `StationObservationManager` , is then defined, which is +from the modified version of the inversion tools provided by Thetis. In this case, as the calibration points have been +defined by a forward run, we can use `load_observation_data` to interpolate the observation time series to the model +time and also stores the time series data to disk. The station manager will also include the variable that is being +optimised for, in this case, the velocity components. + +We can then set up the inversion manager, `InversionManager`, again from the modified version of the inversion tools of +Thetis. The station manager is the first argument and the two key other arguments are the penalty parameters and cost +scaling. The cost function is regularised by an additional term, in this case, the Hessian (second derivative) of the +elevation field. This prevents overfitting of the Manning field i.e. having a highly variable field. The penalty +parameters are the regularisation parameters that control the strength of the regularisation. Higher values increase the +weight of the penalty term, leading to a smoother friction field, while lower values allow more variability. The cost +scaling normalises the regularisation term by the local mesh element size, so that the degree of penalization adapts to +the local mesh resolution. In regions with finer mesh resolution, the scaling ensures that higher variability in +friction is allowed, whilst in regions of lower resolution less variability is allowed to prevent overfitting to sparse +data points. The cost function is then defined using the inversion manager, which is the Hessian regularised L2 norm. +The actual class, `HessianRecoverer2D`, for calculating this loss can be found in `thetis.diagnostics.py`. + +The forward model is then run (`solver_obj.iterate`), also passing `export_func=cost_function` to allow us to alter +the boundary conditions and perform an important step for the adjoint. Passing `export_func=cost_function` effectively +embeds the dependency of the model state on the control variables into the cost function, forming the reduced +functional. The reduced functional is thus the original cost function plus the regularization term, modified by the +model equations. Running the model gives us the baseline cost and more importantly, calculates the gradients of the cost +function with respect to the control variables (friction) which are fed into the adjoint method. The annotation process +is then stopped, which has been recording the computations related to the cost function and its derivatives. + +Optimisation parameters are then defined, which are the maximum number of iterations and the tolerance for the +optimisation convergence criterion (threshold for the relative change in the cost function, below which the optimisation +process will terminate). The optimisation is run by calling `inv_manager.minimize`. The L-BFGS-B algorithm is used as it +is suitable for bound-constrained problems, which are specified earlier alongside the penalty parameters for the cost +functional. These are the minimum and maximum values of bed friction allowed. + +The remainder of the script performs file saving and preparation for visualisation in ParaView. + +### Constant Bed Friction + +```sh +source ~/firedrake/bin/activate +make invert CASE=Constant +``` + +For a constant bed friction, there are some differences which are enforced by changing the case entry, as explained +below. + +Firstly, we do not need penalty parameters in the inversion problem as we will not have any variation across the field +and thus there is no smoothing required. Now, instead of adding the Manning field as a `Control`, we will define the +friction through a `Constant`, and then project this `Constant` onto the Manning field. This `Constant` then becomes our +`Control`, and as it is a `Constant`, it cannot vary. This is inherently dealt with by `Firedrake` and `pyadjoint`. + +The standard implementation of `InversionManager` in Thetis is not flexible in dealing with the `Control` not being +a `Function` i.e. the Manning, bathymetry etc., so we this is where modifications start to come in. +We need to export the Manning at each iteration, so we need to extract the mesh from the `StationObservationManager`, +extract the function space and create a `Function` to assign the `Control` value to. This can then be exported to `.vtu` +at each iteration. + +### Region Based Bed Friction + +```sh +source ~/firedrake/bin/activate +make invert CASE=Regions +``` + +For region-based bed friction, the `InversionManager` has again been modified so that we can export things +correctly. Here, we need to create a mapping that relates the Manning values to the regions of the mesh. Again, there is +no need for penalising the Hessian. We also need to use a `Constant` for each area, and each `Constant` is one +`Control`. The adjoint works on graphical connections, so we need to ensure that the adjoint process can back-propagate +through the various layers to our `Controls`. This is why we need to store our values as `Functions` on our `mesh` +`FunctionSpace`. + +We can define four regions, for example, in the following manner: + +``` +mask_values = [np.logical_and(x < 50e3, y < 6e3).astype(float), + np.logical_and(x < 50e3, y >= 6e3).astype(float), + np.logical_and(x >= 50e3, y < 6e3).astype(float), + np.logical_and(x >= 50e3, y >= 6e3).astype(float)] + +# Create Function objects to store the coefficients +m_true = [Constant(i+1., domain=mesh2d) for i in range(len(mask_values))] +masks = [Function(V) for _ in range(len(m_true))] + +# Assign the mask values to the Functions +for i, mask_ in enumerate(masks): + mask_.dat.data[:] = mask_values[i] +``` + +Importantly, the masks i.e. our regions, will remain consistent. The only thing that will change will be the values +associated with each mask. This means we can define each mask by assigning its values from `NumPy` operators using +`mask.dat.data[:] = mask_values[i]`. In the case of bed particle size mapping, this is important, because it would be +challenging to have to define a series of `conditional` or other operators to define each area. Note that if we +included this assignment of the masks at each iteration of the adjoint, it would not work as there is no graphical +connection when we do assignments with `function.dat.data[:] = values`. We can then define our mapping function as +follows: + +``` +def update_n(n, m): + # Reset n to zero + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, masks): + n += m_ * mask_ +``` + +We then iterate through each mask (region) and then add the corresponding value of Manning (n) friction. We now have a +mapping that `PyAdjoint` can understand. + +As for the `InversionManager`, we can then provide this mapping `update_n` function which allows us to export the +`Control` and `Gradient` fields correctly, rather than having `m` outputs for `m` controls. We can then run the forward, +inverse and plotting scripts in order. + +### Independent Point Scheme + +```sh +source ~/firedrake/bin/activate +make invert CASE=IndependentPointsScheme +``` + +The independent point scheme approach works in the same way as the region-based approach, where we have a mapping +function which tells us how the Manning field changes with respect to our input independent point values. We can use the +same `InversionManager` updates, and the only thing we need to do is change the masks we generate. For a linear +interpolation of the points, this mapping is generated as follows: + +``` +# Get domain limits, define independent points and specify their values +lx, ly = np.max(x), np.max(y) +points = [(lx/4, ly/4), (lx/4, 3*ly/4), (lx/2, ly/4), (lx/2, 3*ly/4), + (3*lx/4, ly/4), (3*lx/4, 3*ly/4)] +m_true = [Constant(0.01*(i+1), domain=mesh2d) for i in range(len(points))] +M = len(m_true) + +# Use Python's numpy to create arrays for the interpolation points +interp_x = np.array([p[0] for p in points]) +interp_y = np.array([p[1] for p in points]) +points = np.column_stack((interp_x, interp_y)) + +# Create the interpolators, use nearest neighbour interpolation outside the convex +# hull of the linear interpolator +linear_interpolator = LinearNDInterpolator(points, np.eye(len(points))) +nearest_interpolator = NearestNDInterpolator(points, np.eye(len(points))) + +# Apply the interpolators to the mesh coordinates to get linear coefficients +# (these do not depend on the magnitude of the points) +linear_coefficients = linear_interpolator(coordinates) +nan_mask = np.isnan(linear_coefficients).any(axis=1) +linear_coefficients[nan_mask] = nearest_interpolator(coordinates[nan_mask]) + +# Create Function objects to store the coefficients +masks = [Function(V) for _ in range(len(points))] + +# Assign the linear coefficients to the masks +for i, mask in enumerate(masks): + mask.dat.data[:] = linear_coefficients[:, i] +``` + +Now, instead of masks with 0/1 values, we have masks which describe the contribution of each point to the rest of the +domain. Note that this will only work for linear interpolation, as we cannot generate static coefficients for non-linear +mappings (RBF, quadratic, cubic etc.). In those cases, we would need to 'annotate' the interpolation functions for +`PyAdjoint` to track the gradient through. + +## Post-processing + +```sh +source ~/firedrake/bin/activate +make plot CASE=NodalFreedom +make plot CASE=Constant +make plot CASE=Regions +make plot CASE=IndependentPointsScheme +``` + +To plot the progress, we can use the Makefile to run `plot_velocity_progress.py`. This plots the velocity over time at +each of the station locations for each iteration of the optimisation, relative to the ground truth from the forward run. diff --git a/examples/headland_inversion/headland.geo b/examples/headland_inversion/headland.geo new file mode 100644 index 000000000..68d2c3e4f --- /dev/null +++ b/examples/headland_inversion/headland.geo @@ -0,0 +1,51 @@ +basin_x = 20000; +basin_y = 6000; + +headland_x_scale = 0.2; +headland_y = 2000; + +site_x = 1600; +site_y = 1000; +site_x_start = basin_x/2-site_x/2; +site_x_end = basin_x/2+site_x/2; + +site_y_start = basin_y/2 - 500; +site_y_end = site_y_start+site_y; + +element_size = 150; +element_size_coarse = 400; + +Point(1) = {0, 0, 0, element_size_coarse}; +Point(2) = {basin_x, 0, 0, element_size_coarse}; + + +// Generate nodes for the headland +res = 100; +For k In {0:res:1} + x = basin_x/res*k; + b = 100; + y = basin_y - headland_y*Exp(-0.5*((headland_x_scale*(x-basin_x/2))/b)^2); + Point(10+k) = {x, y, 0, element_size_coarse}; +EndFor + +// Generate lines for the headland + +BSpline(100) = { 10 : res+10 }; + +Line(101) = {10, 1}; +Line(102) = {1, 2}; +Line(103) = {2, res+10}; +Line Loop(104) = {100, -103, -102, -101}; + +// Generate site nodes +Point(1000) = {site_x_start, site_y_start, 0, element_size}; +Extrude{site_x, 0, 0} { Point{1000}; Layers{site_x/element_size}; } +Extrude{0, site_y, 0} { Line{105}; Layers{site_y/element_size}; } +Line Loop(110) = {106, -108, -105, 107}; +Plane Surface(111) = {104, 110}; +Plane Surface(112) = {110}; +Physical Line(1) = {101}; +Physical Line(2) = {103}; +Physical Line(3) = {100, 102}; +Physical Surface(1) = {111}; +Physical Surface(2) = {112}; diff --git a/examples/headland_inversion/headland.msh b/examples/headland_inversion/headland.msh new file mode 100644 index 000000000..1a9b74e77 --- /dev/null +++ b/examples/headland_inversion/headland.msh @@ -0,0 +1,4042 @@ +$MeshFormat +2.2 0 8 +$EndMeshFormat +$Nodes +1345 +1 0 0 0 +2 20000 0 0 +3 0 6000 0 +4 20000 6000 0 +5 9200 2500 0 +6 10800 2500 0 +7 9200 3500 0 +8 10800 3500 0 +9 394.7231463025948 6000 0 +10 789.4462926047386 6000 0 +11 1184.16943890688 6000 0 +12 1578.89258520935 6000 0 +13 1973.615731511654 6000.000000000001 0 +14 2368.338877813801 6000 0 +15 2763.062024115772 6000 0 +16 3157.785170417816 6000 0 +17 3552.508316719734 6000 0 +18 3947.231463021652 6000 0 +19 4341.954609323481 6000 0 +20 4736.677755624955 6000 0 +21 5131.400901926971 6000 0 +22 5526.124048228285 6000.000000000001 0 +23 5920.847194529607 5999.999999999966 0 +24 6315.570340830852 5999.999999988076 0 +25 6710.293487132288 5999.999997764712 0 +26 7105.016633432545 5999.99976766648 0 +27 7499.739779146791 5999.986619091436 0 +28 7894.462552301992 5999.57383611023 0 +29 8289.091266343863 5992.512724028064 0 +30 8677.366129638298 5929.687379862654 0 +31 8999.384086147591 5708.845030124471 0 +32 9221.418317957163 5383.544671949331 0 +33 9394.375602989896 5028.845477912363 0 +34 9552.63630318427 4667.240061701966 0 +35 9724.867311589362 4312.245283507988 0 +36 9999.999999965004 4051.255769075576 0 +37 10275.13268837701 4312.245283448618 0 +38 10447.36369678898 4667.240061641462 0 +39 10605.62439698389 5028.845477853859 0 +40 10778.58168201302 5383.544671894886 0 +41 11000.61591381219 5708.84503007993 0 +42 11322.63387030728 5929.68737984345 0 +43 11710.90873360071 5992.51272402536 0 +44 12105.53744764518 5999.573836110048 0 +45 12500.26022080309 5999.986619091429 0 +46 12894.98336652209 5999.999767666479 0 +47 13289.7065128265 5999.999997764712 0 +48 13684.42965913181 5999.999999988076 0 +49 14079.15280543647 5999.999999999965 0 +50 14473.87595174091 6000 0 +51 14868.59909804579 6000 0 +52 15263.3222443507 6000.000000000001 0 +53 15658.04539065491 6000 0 +54 16052.76853695947 6000 0 +55 16447.49168326367 6000 0 +56 16842.21482956756 6000 0 +57 17236.93797587379 6000 0 +58 17631.66112217813 6000 0 +59 18026.38426848068 6000 0 +60 18421.1074147835 6000 0 +61 18815.83056108795 6000 0 +62 19210.55370739146 6000 0 +63 19605.27685369587 5999.999999999999 0 +64 0 5600.000000003221 0 +65 0 5200.000000002534 0 +66 0 4800.000000005755 0 +67 0 4400.000000008977 0 +68 0 4000.000000012198 0 +69 0 3600.000000012683 0 +70 0 3200.000000012873 0 +71 0 2800.000000013062 0 +72 0 2400.000000013251 0 +73 0 2000.000000013441 0 +74 0 1600.000000011369 0 +75 0 1200.000000008527 0 +76 0 800.0000000056843 0 +77 0 400.0000000028413 0 +78 399.9999999996764 0 0 +79 799.9999999992851 0 0 +80 1199.999999998882 0 0 +81 1599.999999998817 0 0 +82 1999.999999998839 0 0 +83 2399.999999998829 0 0 +84 2799.999999998201 0 0 +85 3199.999999997313 0 0 +86 3599.999999996426 0 0 +87 3999.999999995538 0 0 +88 4399.99999999524 0 0 +89 4799.999999995262 0 0 +90 5199.999999995284 0 0 +91 5599.999999995304 0 0 +92 5999.999999995327 0 0 +93 6399.999999995348 0 0 +94 6799.99999999537 0 0 +95 7199.999999995392 0 0 +96 7599.999999995413 0 0 +97 7999.999999995437 0 0 +98 8399.999999994563 0 0 +99 8799.999999992764 0 0 +100 9199.999999990969 0 0 +101 9599.999999989172 0 0 +102 9999.999999987729 0 0 +103 10399.99999998922 0 0 +104 10799.99999999106 0 0 +105 11199.9999999929 0 0 +106 11599.99999999474 0 0 +107 11999.99999999658 0 0 +108 12399.99999999842 0 0 +109 12800.00000000026 0 0 +110 13200.0000000021 0 0 +111 13600.00000000394 0 0 +112 14000.00000000578 0 0 +113 14400.00000000763 0 0 +114 14800.00000000946 0 0 +115 15200.00000001131 0 0 +116 15600.00000001315 0 0 +117 16000.00000001499 0 0 +118 16400.00000001615 0 0 +119 16800.00000001438 0 0 +120 17200.00000001258 0 0 +121 17600.00000001078 0 0 +122 18000.00000000899 0 0 +123 18400.00000000719 0 0 +124 18800.00000000539 0 0 +125 19200.0000000036 0 0 +126 19600.0000000018 0 0 +127 20000 399.9999999989609 0 +128 20000 799.9999999981457 0 +129 20000 1199.99999999639 0 +130 20000 1599.999999994241 0 +131 20000 1999.999999992092 0 +132 20000 2399.999999991311 0 +133 20000 2799.999999990677 0 +134 20000 3199.999999990045 0 +135 20000 3599.999999989412 0 +136 20000 3999.999999988778 0 +137 20000 4399.999999990407 0 +138 20000 4799.999999992805 0 +139 20000 5199.999999995203 0 +140 20000 5599.999999997602 0 +141 9360 2500 0 +142 9520 2500 0 +143 9680 2500 0 +144 9840 2500 0 +145 10000 2500 0 +146 10160 2500 0 +147 10320 2500 0 +148 10480 2500 0 +149 10640 2500 0 +150 9360 3500 0 +151 9520 3500 0 +152 9680 3500 0 +153 9840 3500 0 +154 10000 3500 0 +155 10160 3500 0 +156 10320 3500 0 +157 10480 3500 0 +158 10640 3500 0 +159 9200 2666.666666666667 0 +160 9200 2833.333333333333 0 +161 9200 3000 0 +162 9200 3166.666666666667 0 +163 9200 3333.333333333333 0 +164 10800 2666.666666666667 0 +165 10800 2833.333333333333 0 +166 10800 3000 0 +167 10800 3166.666666666667 0 +168 10800 3333.333333333333 0 +169 6284.356970682712 3083.333333333322 0 +170 13716.54516714059 3096.419372872169 0 +171 3264.821194432518 3026.647936139658 0 +172 16735.17880556764 3026.64793613909 0 +173 7674.618235290118 2590.07888449867 0 +174 12309.7202377597 2595.546547200407 0 +175 7683.12879349145 4296.537537175147 0 +176 12316.87120650696 4296.537537172547 0 +177 9760 1246.159999999306 0 +178 8385.936955490535 1384.440855511552 0 +179 10722.07727456057 1517.555669771336 0 +180 8248.070553022812 3418.278319587157 0 +181 11752.38455817361 3418.398522693478 0 +182 4839.956462508089 1884.064068386601 0 +183 15160.04353749206 1884.064068386427 0 +184 4753.750344961758 4140.101209056225 0 +185 15246.2496550387 4140.101209056021 0 +186 1862.599645299243 4202.752594217917 0 +187 18137.40035470145 4202.75259421762 0 +188 18181.43549468189 1850.524787634464 0 +189 1818.564505317294 1850.52478763479 0 +190 6538.981372867257 1549.047925723278 0 +191 13482.61489871375 1594.863051871933 0 +192 11986.39450952087 1290.274318622902 0 +193 6244.491411002624 4542.260315475031 0 +194 13755.50858899609 4542.260315475911 0 +195 8656.616919835751 4307.663410221223 0 +196 11343.38308016363 4307.663410220265 0 +197 3299.653990325279 4507.044311738799 0 +198 16700.34600967553 4507.044311737314 0 +199 3406.384969045153 1542.545615909071 0 +200 16593.61503095514 1542.545615908351 0 +201 8420.406155424962 2571.508458244021 0 +202 11547.43582258716 2583.333333333333 0 +203 9260.243998808279 1819.55963952131 0 +204 10084.22400367152 1879.069431862951 0 +205 7264.262005058716 3415.798944382279 0 +206 12736.63007091874 3435.591896378926 0 +207 11237.95481337376 1965.51112570562 0 +208 18726.35696986066 2999.256648782369 0 +209 1273.643030141728 2999.256648783081 0 +210 5185.57302223415 3050.622126649283 0 +211 14814.42697776616 3050.622126649168 0 +212 8663.689162189865 3083.333333333331 0 +213 11336.31083781016 3083.333333333331 0 +214 7488.078224335504 1659.786195627216 0 +215 8821.075131647933 2076.934663566055 0 +216 10560 1999.028993424386 0 +217 5544.940313380364 1080.188991207701 0 +218 14442.24451044545 1076.282958076996 0 +219 1075.956855736462 4922.511169896472 0 +220 18924.04314426468 4922.511169896728 0 +221 18880.22653261046 1096.12221507142 0 +222 1119.77346738903 1096.122215071459 0 +223 11935.18993165957 1994.927342008657 0 +224 8207.714105663417 5038.059586617981 0 +225 11792.28589433572 5038.059586617263 0 +226 5775.373900180055 2212.883433081931 0 +227 14225.28783902244 2210.483570537058 0 +228 5315.320984388062 4968.443912393563 0 +229 14684.67901561157 4968.443912390699 0 +230 11353.20385460036 867.4238444644412 0 +231 2272.921302565075 3199.894902432845 0 +232 17727.07869743636 3199.894902431999 0 +233 4272.169276660456 1019.475124306973 0 +234 15727.83072333975 1019.475124307473 0 +235 4198.809333121072 4955.362796869053 0 +236 15801.19066687698 4955.362796870008 0 +237 7090.66242275942 5045.188700850005 0 +238 12909.33757723751 5045.188700849987 0 +239 9028.360124597149 1059.047134473799 0 +240 17544.68422219015 1022.313215287765 0 +241 2455.315777810033 1022.31321528848 0 +242 8102.87449640104 2001.469370581603 0 +243 2412.276869477229 4990.450849175756 0 +244 17587.72313052323 4990.450849175242 0 +245 12702.15819689797 1908.310948941726 0 +246 6805.668694084289 2429.518886116075 0 +247 10400.80637490991 787.839003612712 0 +248 4182.208389062731 2674.443799329761 0 +249 15817.79161093745 2674.443799329434 0 +250 12823.75116404658 905.4255621538698 0 +251 8709.713771288516 3596.368973622488 0 +252 11290.28622871152 3596.368973622454 0 +253 5662.001126883189 3796.335457315276 0 +254 14337.99887311729 3796.335457316371 0 +255 3931.962161204927 3729.513228031544 0 +256 16068.03783879501 3729.513228030842 0 +257 2673.339224035349 2167.030485192308 0 +258 17326.66077596487 2167.030485192082 0 +259 7793.894102644362 874.2555976788888 0 +260 914.6436602505075 3845.706826220245 0 +261 19085.35633974739 3845.706826215103 0 +262 19084.56161442051 2099.921767596168 0 +263 915.4383855799589 2099.921767597761 0 +264 13133.47117566762 2612.49613496122 0 +265 9588.206128226337 2091.912135437683 0 +266 10725.5684977011 4044.310357023263 0 +267 9274.43150229511 4044.310357021802 0 +268 6898.593959724585 4155.389102614685 0 +269 13101.22931874936 4155.846120437988 0 +270 6907.288132713206 738.6706847563797 0 +271 2890.035921994789 3701.787413066201 0 +272 17109.96407800608 3701.787413065442 0 +273 8853.88534732792 2602.529550360484 0 +274 11155.42659921288 2569.925596932934 0 +275 13708.10898108331 790.4582959463701 0 +276 10265.60085304795 2196.929155209989 0 +277 8493.781245774542 739.9574729527853 0 +278 8076.530683781437 2927.683147422959 0 +279 11923.46931621871 2927.683147422958 0 +280 11200.85699077186 1399.243110683489 0 +281 4008.740866515749 1957.70809069519 0 +282 15991.25913348443 1957.70809069477 0 +283 10243.40361154333 1364.667479013518 0 +284 9523.185053774527 660.174685627685 0 +285 3355.146743568775 5265.53595680674 0 +286 16644.8532564156 5265.53595680551 0 +287 9285.272295665292 2193.335366806563 0 +288 3572.373754178079 731.5262929233667 0 +289 16427.62624582126 731.5262929227259 0 +290 5975.094561704406 5306.074153678674 0 +291 14024.90543829697 5306.074153677972 0 +292 18267.02734707235 695.9196284940658 0 +293 1732.972652928512 695.9196284950265 0 +294 9935.893239061079 2185.510971058263 0 +295 8102.401140844751 3998.62066549926 0 +296 11897.59885915421 3998.620665498356 0 +297 1753.918005205195 5314.84194605938 0 +298 18246.08199479417 5314.841946058248 0 +299 1893.780937647645 2585.86136602907 0 +300 18106.2190623548 2585.861366028761 0 +301 6175.996762578421 647.5688774148955 0 +302 11853.15944577599 598.2828021485011 0 +303 4825.261908174642 635.7652312851869 0 +304 15052.62598198908 689.4530046167221 0 +305 4558.835727370568 3408.389130764352 0 +306 15441.16427262962 3408.389130764089 0 +307 10900.44626016274 2181.635086575088 0 +308 8811.96538731589 1585.727600181352 0 +309 3385.274338677079 2314.523964685669 0 +310 16614.72566132287 2314.52396468513 0 +311 7152.948680591324 2800.473807601176 0 +312 6344.26731380839 3763.745001668938 0 +313 13609.67338331098 3760.862778559018 0 +314 9682.489609719763 1714.610562768547 0 +315 9630.659379846353 3833.123726228419 0 +316 10400 3853.606310017917 0 +317 7759.13255757541 3663.796200649027 0 +318 12239.34725022461 3664.86983399855 0 +319 8737.590878796449 4871.237152684922 0 +320 11262.40912119997 4871.237152685895 0 +321 2477.470400929725 4318.029606242414 0 +322 17522.52959907106 4318.029606241083 0 +323 13582.52535128111 2180.913512661644 0 +324 7562.601069583045 5407.309474031225 0 +325 12437.39893040703 5407.309474025365 0 +326 12346.11721300648 3080.373010515299 0 +327 4797.792629382606 5379.167665320789 0 +328 15202.20737061528 5379.167665319863 0 +329 616.7359223045594 3238.590333431647 0 +330 19383.26407769694 3238.590333426163 0 +331 1685.759329945808 3515.02423418906 0 +332 18314.2406700558 3515.024234188418 0 +333 7327.539575914996 2194.249278168166 0 +334 8179.494822215057 4551.487576822238 0 +335 11820.50517778399 4551.48757682175 0 +336 4137.123681913746 4344.886057649778 0 +337 15862.87631808739 4344.886057650489 0 +338 8912.45984406915 3229.68775862038 0 +339 8912.421765197347 2916.666666666666 0 +340 11087.54015593087 3229.687758620376 0 +341 11087.57823480267 2916.666666666666 0 +342 7677.144256512141 3066.863620969804 0 +343 5372.118687372103 4339.068788105596 0 +344 14627.88131262908 4339.068788103927 0 +345 633.1819654082834 5416.390459291571 0 +346 19366.81803459276 5416.390459292016 0 +347 5952.147193374804 1655.146767035148 0 +348 14042.39487997202 1567.104820859141 0 +349 19367.25715074659 657.4299813223528 0 +350 632.7428492526444 657.4299813223118 0 +351 635.2709115833328 4478.88970073328 0 +352 19364.72908841634 4478.889700727567 0 +353 8970.540571670983 3823.607475674241 0 +354 11029.45942832903 3823.60747567331 0 +355 19363.89910221289 1483.138059903191 0 +356 636.1008977842556 1483.138059903303 0 +357 5680.3195214876 2762.749840715774 0 +358 14312.65221839849 2785.48379766275 0 +359 10812.69125022083 1068.900901094107 0 +360 5178.427618833997 2401.906503167589 0 +361 14821.57238116622 2401.906503167464 0 +362 11598.34222178475 1706.190224092491 0 +363 17350.54348796298 1586.919878029891 0 +364 2649.456512037144 1586.919878030619 0 +365 17075.92215344822 598.1055444034367 0 +366 2924.077846551565 598.1055444040512 0 +367 598.570309240607 2633.045935986123 0 +368 19401.42969076022 2633.045935982079 0 +369 9387.263360780415 1413.619589538378 0 +370 12552.30776708323 1377.571779546642 0 +371 11368.12404276652 2309.003225312643 0 +372 8073.733092264056 2421.817678840915 0 +373 6593.506604412179 5456.104950385612 0 +374 13406.49339558663 5456.104950389273 0 +375 6340.507318733879 2064.068348012926 0 +376 8959.426470487026 542.886066266462 0 +377 11913.06866110976 2412.101615005036 0 +378 4924.190492030442 1317.233620557065 0 +379 15075.80950796964 1317.233620556771 0 +380 4760.828929209513 4767.563781701442 0 +381 15239.17107079049 4767.563781700844 0 +382 7681.352950990406 4821.184387196901 0 +383 12318.6470490079 4821.184387193864 0 +384 8461.239489354946 1812.771272719722 0 +385 10400.16474504509 1682.615350654203 0 +386 3862.020769476163 3147.106929962194 0 +387 16137.97923052379 3147.106929961514 0 +388 13173.52959159291 3141.623620639338 0 +389 2736.217016545143 2816.970956001085 0 +390 17263.78298345589 2816.970956000863 0 +391 10846.94051756599 509.9122177453326 0 +392 5090.177272633723 3657.25157523624 0 +393 14909.82272736645 3657.25157523633 0 +394 1284.457658152052 4365.36721674919 0 +395 18715.54234184894 4365.36721674945 0 +396 8418.041517070855 2208.070268471316 0 +397 7942.479757210333 1554.446267691946 0 +398 18741.88886082852 1653.745575871329 0 +399 1258.111139170655 1653.745575872029 0 +400 8996.433123609473 2350.988045200276 0 +401 6748.493327684047 3260.994762026218 0 +402 9092.176508448649 4483.014110110006 0 +403 10907.82349155016 4483.014110109433 0 +404 7041.1530984426 1340.531054872649 0 +405 10932.64699708091 1789.281344658924 0 +406 12306.27884654399 2155.492854969914 0 +407 17199.81341288184 5431.20322445928 0 +408 2800.186587109314 5431.203224462774 0 +409 5379.111006687269 1798.65917163233 0 +410 14620.88899331287 1798.65917163217 0 +411 14260.25734178141 578.6202041010756 0 +412 10583.8111157227 2236.58628051282 0 +413 7247.55065461277 4561.665269268055 0 +414 12752.44934538517 4561.665269266654 0 +415 9970.166301014884 861.8893950290826 0 +416 7377.636625757819 3916.049831068015 0 +417 12626.83550227231 3918.438347897993 0 +418 1889.862497026467 4737.918926331009 0 +419 18110.13750297367 4737.918926330194 0 +420 6923.211086035911 1884.245232994878 0 +421 7741.684412506282 2164.402230533762 0 +422 18077.65243092786 1274.355428149813 0 +423 1922.347569072558 1274.355428150556 0 +424 2277.449979451583 3792.319958768161 0 +425 17722.55002054965 3792.319958767246 0 +426 12443.76068595302 521.3497784575623 0 +427 12255.82767008427 1711.760903686192 0 +428 8648.237686325843 2808.559292175373 0 +429 11342.47830580398 2815.965359461365 0 +430 3879.434212468254 5448.769261674292 0 +431 16120.56578752186 5448.769261678926 0 +432 5534.080536359033 565.2117730565902 0 +433 8564.861342496391 5388.095699414802 0 +434 11435.13865749778 5388.095699408339 0 +435 13184.81988251693 520.5331618107017 0 +436 12696.3428035765 2392.629134782302 0 +437 1191.578556318511 5505.838294099386 0 +438 18808.42144368071 5505.838294098978 0 +439 18762.93217966424 549.5130872079952 0 +440 1237.067820336039 549.5130872082266 0 +441 13193.06974725242 3683.542784224433 0 +442 11035.95827194905 2373.825419073627 0 +443 18512.90784695498 2418.733115143028 0 +444 1487.09215304557 2418.733115143589 0 +445 7313.848281686804 519.1144720954082 0 +446 11640.04093296817 1156.568151894189 0 +447 8387.686131627244 2972.866596427259 0 +448 11612.34876504424 2973.333864975194 0 +449 17785.04177409678 612.5972382823188 0 +450 2214.958225905509 612.5972382818723 0 +451 2941.529433095188 4905.81525108713 0 +452 17058.47056690809 4905.81525108094 0 +453 6724.779996820327 4746.317980311591 0 +454 13275.22000317763 4746.317980312775 0 +455 6162.517793118878 2578.64992228956 0 +456 2128.447886905379 5468.807316884946 0 +457 17871.55211309023 5468.807316884271 0 +458 3534.214493464657 4027.075438877941 0 +459 16465.78550653605 4027.075438876894 0 +460 4053.518069560881 546.8123512487213 0 +461 15946.48193043874 546.8123512482138 0 +462 4423.042587684824 1530.785408529081 0 +463 15576.95741231525 1530.785408529251 0 +464 3894.431539339017 1382.683756596734 0 +465 16105.56846066134 1382.683756596538 0 +466 5548.584673163534 5453.665424883605 0 +467 14451.41532683784 5453.665424883402 0 +468 6902.458000756578 3735.721836110532 0 +469 8926.542194361007 3519.959353951325 0 +470 11073.45780563903 3519.959353951211 0 +471 4714.893707795906 2701.608650834709 0 +472 15285.10629220438 2701.608650834504 0 +473 3648.231659026721 4938.919313694611 0 +474 16351.76834096912 4938.919313692798 0 +475 5767.121513833835 4688.431767614969 0 +476 14232.87848616605 4688.431767610829 0 +477 13179.91487064591 2118.222762037277 0 +478 8525.107572519357 3922.030943298922 0 +479 11474.89242748038 3922.030943298087 0 +480 9275.709362188365 3780.704880291203 0 +481 10724.29063781107 3780.704880290773 0 +482 9043.414343144275 1943.839931028625 0 +483 12996.93823010404 1514.845785341798 0 +484 13300.75762774427 1058.116951086669 0 +485 8516.700744074273 3348.744729936785 0 +486 11483.29925592577 3348.744729936794 0 +487 12073.46029618551 3247.566595298092 0 +488 16971.3570763818 1129.104704060696 0 +489 3028.64292361852 1129.104704061006 0 +490 9868.624890963496 1946.848077074253 0 +491 13847.98615316955 2602.783013082168 0 +492 7922.13970015692 3243.263442110007 0 +493 5868.3819778882 3309.155121092026 0 +494 14075.70754892864 3292.461209285405 0 +495 12737.96162765259 2822.387406595255 0 +496 7987.5715451991 483.7711269851098 0 +497 8135.994488467741 995.0304632688192 0 +498 8016.482250184198 5590.680316459512 0 +499 11983.51774980161 5590.680316452405 0 +500 12294.5803651545 1098.286362990924 0 +501 17736.60302268311 2247.920129554323 0 +502 2263.396977317472 2247.92012955398 0 +503 3450.152717132838 3480.07437888983 0 +504 16549.84728286644 3480.074378888597 0 +505 9981.337000638525 1537.042960197024 0 +506 10111.96315390644 370.1964205851398 0 +507 9388.819154369534 1011.356012306316 0 +508 7149.42696261792 5516.759750095744 0 +509 12850.57303737763 5516.759750093889 0 +510 4340.153262158317 2186.850050399615 0 +511 15659.8467378418 2186.850050399336 0 +512 6259.768739578929 1121.710529283964 0 +513 11569.87118760798 2024.510658115728 0 +514 8680.55294189061 2374.16318360577 0 +515 11525.6443933589 478.098042027466 0 +516 9600 2294.957493934717 0 +517 7434.948862017487 1243.621718660672 0 +518 6014.032975345232 4114.022297038508 0 +519 13984.46832284824 4113.92851374856 0 +520 6581.661399553803 433.5675638022429 0 +521 3190.474451015559 1898.363501407939 0 +522 16809.52554898435 1898.363501407413 0 +523 19599.30041536608 3804.137139082004 0 +524 400.6995846313927 3804.137139098979 0 +525 6657.60829357106 2721.274264151801 0 +526 10062.86308008165 3812.579730775791 0 +527 2880.372181710115 3273.177755458408 0 +528 17119.62781829004 3273.177755457512 0 +529 466.0022514817204 4871.073569442931 0 +530 19533.99774851984 4871.073569440914 0 +531 19547.35400608182 2210.061097029875 0 +532 452.6459939191533 2210.061097034302 0 +533 19524.46211371096 1086.869846206467 0 +534 475.5378862877739 1086.86984620743 0 +535 4332.573421577244 3904.093188804255 0 +536 15667.42657842303 3904.093188804053 0 +537 8673.393889862269 1068.982580597247 0 +538 6433.787912281937 4836.842888119279 0 +539 13566.21208771655 4836.842888120083 0 +540 3735.02562396901 2687.656615079218 0 +541 16264.97437603091 2687.656615078766 0 +542 1210.108550534828 3467.042180183112 0 +543 18789.89144946593 3467.042180181811 0 +544 10536.48436372858 1211.251933640656 0 +545 11625.44186995908 2289.12959807064 0 +546 4487.203333118912 443.0456252105388 0 +547 15471.63984144212 443.1392007793627 0 +548 1841.713726761308 3068.616581634665 0 +549 18158.28627324029 3068.616581634229 0 +550 1406.178666946912 3815.810532371825 0 +551 18593.82133305333 3815.810532370661 0 +552 10072.5535305287 2325.740642182353 0 +553 8954.073884973903 4133.910091194283 0 +554 11045.92611502467 4133.910091195026 0 +555 13876.27730955489 1229.089247303101 0 +556 17686.45743739695 1600.251132592277 0 +557 2313.542562603043 1600.251132593117 0 +558 4383.304825156105 5523.489179899751 0 +559 15616.69517483611 5523.489179901615 0 +560 8348.094900267804 3642.40474983937 0 +561 11651.9961219713 3642.428790460259 0 +562 14632.411510768 394.035081582496 0 +563 10331.57407104071 1893.005764603909 0 +564 9010.53430275035 2747.578202808305 0 +565 10988.97871448068 2744.621439074696 0 +566 6450.375927773392 4161.602499033024 0 +567 13541.63886269886 4161.508732211141 0 +568 2949.988397262387 4194.102054559266 0 +569 17050.01160273873 4194.102054558023 0 +570 9118.180002626224 1527.872101337315 0 +571 9028.559732356343 3076.048118021968 0 +572 10754.84522709149 2266.803380007711 0 +573 10971.44026764367 3076.048118021964 0 +574 7365.507929924474 2533.207430754444 0 +575 18493.76125653948 1211.482536667754 0 +576 1506.23874346034 1211.482536668107 0 +577 3064.877510980184 2659.052161580253 0 +578 16935.12248902029 2659.052161579673 0 +579 3589.363216161097 1880.109114358866 0 +580 16410.6367838394 1880.109114358295 0 +581 4457.70983225764 3019.900915700873 0 +582 15542.29016774266 3019.900915700719 0 +583 8194.752980975307 2665.937229902147 0 +584 9760.649851954746 3681.992737585684 0 +585 3697.131327323035 4316.180336332825 0 +586 16302.86867267792 4316.180336331037 0 +587 5301.356883146855 3389.338701863172 0 +588 14698.64311685323 3389.338701863375 0 +589 6649.28939756007 1168.731103828827 0 +590 10408.31855360741 2330.900475113112 0 +591 11802.7899437054 2671.67016481179 0 +592 10978.14597774079 1295.522258545167 0 +593 10625.6575267324 1743.142481300054 0 +594 13468.28205831325 2756.711597025002 0 +595 17729.18978645546 2626.893674049757 0 +596 2270.810213546683 2626.893674049398 0 +597 5150.28080806001 5588.802203185989 0 +598 14849.71919193763 5588.802203186437 0 +599 18520.36241282251 1980.195527978392 0 +600 1479.637587177194 1980.195527979173 0 +601 1507.451450493178 4902.884392340419 0 +602 18492.54854950764 4902.884392338718 0 +603 8509.408532570586 4577.476441961348 0 +604 11490.59146742788 4577.476441961641 0 +605 4034.205632113752 2348.887979166363 0 +606 15965.79436788631 2348.887979165933 0 +607 7810.123167159596 1853.220329867352 0 +608 5820.647267842124 367.5062131532792 0 +609 11933.45050108508 1679.886706260869 0 +610 7821.723040525592 3972.760712582031 0 +611 12178.12183782011 3972.870267004161 0 +612 9491.219678171326 1818.333074696846 0 +613 10047.44848536438 1118.80855378153 0 +614 8085.099661454447 3714.749454941301 0 +615 11914.75700504014 3714.928097824178 0 +616 11196.13297243649 1685.386143506122 0 +617 13423.5231036101 3358.052878078897 0 +618 13810.94263598081 351.1645021777952 0 +619 9411.871367768303 2321.342702623425 0 +620 16687.00141564403 410.6029509979777 0 +621 3312.998584354454 410.6029510001667 0 +622 7843.662202570116 1219.074411626628 0 +623 12823.13183683744 513.5700136046021 0 +624 9013.192746330162 2531.6084659728 0 +625 9786.158911555309 2320.040468056453 0 +626 8214.639259079258 3152.71265203033 0 +627 11789.65341693733 3141.599835608906 0 +628 13753.45666252131 1820.428133529166 0 +629 9717.641533881331 333.1069976254233 0 +630 7628.095018003483 3369.604462905514 0 +631 11164.69201357324 2186.618818685579 0 +632 13769.51217688227 5616.131414195941 0 +633 6230.487823114638 5616.131414192332 0 +634 4694.758053104566 1012.848524824248 0 +635 15280.17291985444 1009.872555240466 0 +636 19649.85826514415 2971.747080916286 0 +637 350.1417348569535 2971.747080930288 0 +638 8674.062728434517 358.4216522111941 0 +639 1029.481060096666 2679.349470317125 0 +640 18970.51893990245 2679.349470316799 0 +641 12204.13708033819 2856.684092083876 0 +642 14115.11933413801 916.6390695540659 0 +643 10040.09090512248 2039.540865619176 0 +644 10791.63729334884 1925.771184566342 0 +645 10474.66217299548 483.7660436085539 0 +646 9142.285249949005 2303.545686438645 0 +647 7313.285396897843 862.8266100266602 0 +648 9442.422640445038 3672.227842904573 0 +649 10561.95727096155 3675.153926302858 0 +650 2591.852522242794 346.4295425965466 0 +651 17408.14747776138 346.4295425964181 0 +652 4149.370345988784 3376.69502463759 0 +653 15850.62965401127 3376.69502463716 0 +654 8768.620773881283 3402.207324781368 0 +655 11231.37922611871 3402.207324781325 0 +656 12361.05041383681 3398.9909674694 0 +657 6461.00258563384 3386.966631106568 0 +658 4410.103879422839 4650.288625399822 0 +659 15589.89612057636 4650.28862539999 0 +660 2522.070423190654 3503.604375382149 0 +661 17477.9295768103 3503.604375381275 0 +662 3149.658272515704 5604.067141611604 0 +663 16850.34172747323 5604.067141609812 0 +664 10401.88748571624 2116.543201791973 0 +665 7803.899984441954 2842.562960285479 0 +666 7033.799635656663 318.4092715832339 0 +667 9302.290743157881 363.9240017559994 0 +668 18150.53017679651 317.5116282567164 0 +669 1849.469823209109 317.5116282570428 0 +670 10952.14364335786 2540.490973898582 0 +671 3855.236873374543 1059.176230695325 0 +672 16144.76312662546 1059.17623069499 0 +673 18177.15966438659 5696.093114577278 0 +674 1822.840335607412 5696.093114577858 0 +675 19669.06769291999 4221.532742845615 0 +676 330.9323070791877 4221.53274286029 0 +677 8206.312860595721 1695.482922072449 0 +678 2329.017940705327 4702.755958045618 0 +679 17670.98205929562 4702.75595804366 0 +680 4781.756223375171 2288.366880805007 0 +681 15218.24377662504 2288.366880804851 0 +682 341.638770800702 1770.065495101885 0 +683 19658.3612291973 1770.065495095293 0 +684 5316.798450878631 3970.559106069507 0 +685 14683.20154912181 3970.559106068668 0 +686 5770.187400693962 815.0869224367752 0 +687 7656.142606690906 325.3006138611291 0 +688 9415.255016810366 2005.916249747326 0 +689 8374.3440481091 4187.574362512677 0 +690 11625.6559518901 4187.574362511938 0 +691 8841.202903618489 1850.352401815671 0 +692 11930.81044148486 944.2443741354814 0 +693 16619.85690108033 1107.91900460444 0 +694 3380.143098920311 1107.919004605188 0 +695 11660.57181363765 808.9234429340158 0 +696 10607.91926803472 4323.391992233952 0 +697 9392.080731948354 4323.3919922402 0 +698 3533.665169407108 5616.261880552182 0 +699 16466.33483058316 5616.261880553735 0 +700 5014.566514845304 297.8727523342393 0 +701 8002.480492179062 4261.850818871182 0 +702 11997.49365421093 4261.869077940138 0 +703 9194.27225510031 790.2340715099298 0 +704 5366.564185134953 1383.615872784357 0 +705 14634.65886657266 1385.223696237558 0 +706 7438.095336257006 5072.890826554025 0 +707 12561.90466373954 5072.890826549778 0 +708 10240 3672.64350973672 0 +709 8758.838119508579 3849.046082794142 0 +710 11241.16188049132 3849.046082793674 0 +711 11727.80829347635 1426.464437630213 0 +712 13226.81577472349 1735.875043429273 0 +713 12178.44169525012 338.9233912963941 0 +714 5626.96138494405 5036.692923699615 0 +715 14373.03861505648 5036.692923697283 0 +716 6440.682844275174 2425.438746728478 0 +717 12851.59616729285 3129.237851493222 0 +718 8903.522792635402 1314.647459979481 0 +719 16295.89688912579 347.5484277533863 0 +720 3704.103110876633 347.5484277548107 0 +721 4919.755830965966 3356.16559378055 0 +722 15080.24416903415 3356.165593780535 0 +723 7497.979019094864 2816.472552182555 0 +724 5547.349319380346 2473.415784212667 0 +725 14451.91826883454 2476.538070115468 0 +726 18170.67840542043 2179.868508705641 0 +727 1829.321594579478 2179.868508706044 0 +728 5161.18981641511 982.2051940039566 0 +729 14827.76378943031 1024.553052548123 0 +730 2800.018629457918 4555.200699926723 0 +731 17199.98137054347 4555.200699922581 0 +732 7361.132129089683 3105.110970712393 0 +733 17932.41256616828 952.0933928535221 0 +734 2067.587433832208 952.0933928541606 0 +735 11349.18590315384 2488.119192696006 0 +736 11173.26453783581 525.0923854958904 0 +737 19096.43860815229 366.2025694587051 0 +738 903.5613918475497 366.2025694596497 0 +739 10710.14374736068 2123.536572918433 0 +740 5088.435262699306 4652.153626206065 0 +741 14911.5647373009 4652.153626204716 0 +742 2111.319137397073 5074.046620813564 0 +743 17888.68086260185 5074.046620812838 0 +744 11122.07208239404 1105.855886713154 0 +745 16684.69644715191 4893.693865675926 0 +746 3315.303552847573 4893.693865679302 0 +747 4735.776630288844 3768.949918208359 0 +748 15264.2233697113 3768.949918208237 0 +749 17522.76602242196 1895.948390485847 0 +750 2477.233977577954 1895.948390486104 0 +751 5984.062399400173 3650.030639240625 0 +752 14012.090823011 3649.666052873381 0 +753 893.3207733169643 5654.044291168641 0 +754 19106.67922668183 5654.044291169009 0 +755 6264.744020294032 1758.030140052567 0 +756 15041.19610916631 331.3093486847005 0 +757 6206.526073472964 301.9344419090883 0 +758 7252.360931855686 1911.928075368965 0 +759 5795.348498139143 5670.015185001383 0 +760 14204.65150185401 5670.015185003957 0 +761 7296.56834778935 4246.021609947794 0 +762 12703.48147433284 4247.449513076889 0 +763 18446.24173451218 2785.78520365133 0 +764 1553.758265490076 2785.785203652074 0 +765 10181.69785097784 1635.875692927965 0 +766 5709.277594203591 4351.905225461553 0 +767 14290.63137677688 4351.899529194425 0 +768 12176.89404173253 2389.944768799768 0 +769 6047.578681464921 4883.6059269343 0 +770 13952.4213185349 4883.605926933125 0 +771 10483.23112717077 1454.505472365023 0 +772 7853.791107888428 4564.124058635741 0 +773 12146.20889211023 4564.12405863208 0 +774 2580.905475915791 3947.18851883727 0 +775 17419.09452408534 3947.188518836418 0 +776 314.1425646808526 3400.000000012777 0 +777 19685.85743532472 3399.999999989727 0 +778 9894.35664236616 1732.667538639405 0 +779 1636.577047387175 4475.212516349821 0 +780 18363.42295261362 4475.212516349748 0 +781 8906.980859445644 5239.435623543384 0 +782 11093.01914053652 5239.435623519698 0 +783 9920 3649.045896593092 0 +784 4523.381713847209 5088.073486269308 0 +785 15476.61828614959 5088.073486269701 0 +786 10320.29903742458 1063.042874643458 0 +787 8463.036075905275 3145.616727046357 0 +788 11536.96392409476 3145.616727046343 0 +789 12055.01321347738 2642.271722553972 0 +790 18590.17732588467 876.5740870649021 0 +791 1409.822674114953 876.5740870651788 0 +792 9606.746374449227 3637.468861343735 0 +793 10400.39145419231 3640.280749211499 0 +794 17487.02951879989 5672.039595698043 0 +795 2512.970481192519 5672.039595698524 0 +796 7894.224429579221 5154.128243627179 0 +797 12105.77557041627 5154.128243624199 0 +798 7944.472378386767 2631.769779283599 0 +799 8300.073446096612 2411.755023198215 0 +800 9107.490485514418 4819.943089237285 0 +801 10892.50951448409 4819.94308922556 0 +802 9025.637436779096 1746.389948383854 0 +803 6675.584951385474 2124.634314271225 0 +804 9427.485974611593 1622.111285824235 0 +805 3937.837964904882 4701.583602471787 0 +806 16062.16203509531 4701.583602471056 0 +807 362.7688008585696 5636.774846645788 0 +808 19637.23119914248 5636.774846646468 0 +809 1857.458604411645 3874.117190889763 0 +810 18142.54139558927 3874.117190889198 0 +811 12838.38880885378 3676.674942271188 0 +812 983.820161589042 4570.444961294786 0 +813 19016.17983841206 4570.444961295014 0 +814 8427.138047806515 5649.627944698555 0 +815 11572.86195216532 5649.627944689635 0 +816 8654.69115568634 2588.951351552448 0 +817 7532.161392219901 2012.383245281233 0 +818 8863.359735466742 851.036829463239 0 +819 4967.748499032394 5045.770938347794 0 +820 15032.25150096614 5045.770938346388 0 +821 8835.513343587492 3079.93427267637 0 +822 9003.222180604538 3365.984399456511 0 +823 10996.77781939546 3365.984399456495 0 +824 11164.48665641254 3079.93427267635 0 +825 19058.85989021316 1751.80901460403 0 +826 941.1401097863018 1751.809014606536 0 +827 10949.91497755844 1540.158564389872 0 +828 19648.33721924721 357.1398175544452 0 +829 351.6627807522109 357.1398175542363 0 +830 12458.05957271879 2825.057312163598 0 +831 11000.23104423336 807.6948671096154 0 +832 13341.95714261233 2422.002916579605 0 +833 19713.55890597124 4554.299202599462 0 +834 286.4410940288482 4554.299202610247 0 +835 300.5257672961346 1414.556127408389 0 +836 19699.47423270053 1414.556127403625 0 +837 5442.034140001918 2142.265534001469 0 +838 14557.95744659554 2142.351536782082 0 +839 9092.834349680497 3661.398145627083 0 +840 10907.16565031948 3661.398145626495 0 +841 6830.461146253802 5676.923175849441 0 +842 13169.53885373048 5676.923175853118 0 +843 6434.837128949886 845.0812650850499 0 +844 12327.70198783855 5741.960337559164 0 +845 7672.29801212913 5741.9603375642 0 +846 2191.38985663052 4123.309271145437 0 +847 17808.61014337042 4123.309271144684 0 +848 6726.434892515511 5065.831851854479 0 +849 13273.56510748305 5065.83185185636 0 +850 11208.69126332762 2394.570422325476 0 +851 12863.61741043586 1228.15902227875 0 +852 7466.277015260415 3652.470847028749 0 +853 12532.52470068383 3654.971001162192 0 +854 8815.277315055944 2766.818752948073 0 +855 11189.31798029398 2773.323448502822 0 +856 9715.910167265996 967.5603416271395 0 +857 19058.87435655199 3143.662130782638 0 +858 941.1256434494722 3143.662130783786 0 +859 9752.686584754343 2152.192492064084 0 +860 9068.284891692609 2916.666666666666 0 +861 10931.71510830742 2916.666666666666 0 +862 6911.596737088104 2662.316100588404 0 +863 8505.363807898808 5086.578629159958 0 +864 11494.63619209803 5086.57862915765 0 +865 5851.328169155106 1259.129061784175 0 +866 10689.96723942036 769.973202186099 0 +867 9930.436428882898 2363.819405407526 0 +868 7182.400369068864 3683.438766230663 0 +869 7539.014734050532 4569.438446015296 0 +870 12467.27984576086 4561.445827420564 0 +871 8033.799139488223 4818.808844626914 0 +872 11966.20086051075 4818.808844625668 0 +873 10222.47595453068 2344.938039914868 0 +874 11030.01854450244 1973.169700553383 0 +875 7560.002963962344 2346.205888049995 0 +876 9598.946888032518 1456.703223941862 0 +877 18391.91833163432 1558.312182438279 0 +878 1608.081668365216 1558.312182438814 0 +879 19052.82229824859 748.9539418751228 0 +880 947.1777017504138 748.9539418743631 0 +881 5065.640406914174 1652.274963657522 0 +882 14934.35959308601 1652.274963657141 0 +883 8580.597333205304 1996.695387426693 0 +884 5697.196251981619 1898.667784909722 0 +885 14213.85124966329 1891.663514906008 0 +886 2048.118355392028 3478.943817822191 0 +887 17951.88164460926 3478.943817821425 0 +888 8535.570524631077 1589.275443812125 0 +889 11831.44910687526 283.0608470944723 0 +890 909.3843136732356 4229.777682917031 0 +891 19090.6156863262 4229.777682916307 0 +892 12510.46817254991 1663.376993464728 0 +893 13916.39855363433 2236.734438733896 0 +894 2141.436798508838 2892.781053594786 0 +895 17858.56320149347 2892.781053595185 0 +896 9048.17804576512 2160.018026364641 0 +897 5371.824758756657 2781.479465012886 0 +898 14625.445451929 2768.568851940252 0 +899 9533.812788740597 4077.094012853915 0 +900 10469.704385736 4067.150617356477 0 +901 7014.40370949721 2161.608206757236 0 +902 954.039494130182 5262.413168016759 0 +903 19045.96050587074 5262.413168017152 0 +904 14544.03170637397 771.462056050828 0 +905 19023.67662822225 1397.068546650917 0 +906 976.3233717776191 1397.068546651938 0 +907 4636.244774756323 5736.035735643316 0 +908 15363.7552252318 5736.035735642206 0 +909 8814.710448319891 4530.537961957042 0 +910 11185.2895516789 4530.537961954879 0 +911 2599.295710885186 710.3904826203158 0 +912 17400.70428911386 710.3904826215551 0 +913 8316.676957280879 356.670432182972 0 +914 4466.213070404304 1854.92869100917 0 +915 15533.78692959585 1854.928691008947 0 +916 6023.083707487211 1978.641015847199 0 +917 8105.088925016134 1315.231192435472 0 +918 8443.058505835659 2771.173176297067 0 +919 11542.764312513 2781.400482083056 0 +920 8196.933743074243 2210.650784162184 0 +921 6691.551615652336 3921.678327008788 0 +922 7683.82056098166 1464.550233571047 0 +923 14226.27590182894 1287.844911246491 0 +924 13283.1179174876 3942.800330818548 0 +925 5045.790797332998 4251.104199784721 0 +926 14954.20920266756 4251.104199783989 0 +927 853.0413388434372 3448.288505566445 0 +928 19146.95866115489 3448.288505562338 0 +929 13036.12291492013 3404.120662181 0 +930 5097.848304428274 2075.023719680956 0 +931 14902.15169557192 2075.023719680859 0 +932 17520.05527718442 1362.832767644254 0 +933 2479.944722815294 1362.832767645212 0 +934 17025.23283081027 273.4091459520607 0 +935 2974.767169193404 273.4091459537085 0 +936 13767.50800520026 3431.492458333774 0 +937 13429.78868193094 387.6090423393708 0 +938 8502.915682980671 2391.845839384795 0 +939 4471.776955380793 4321.805704615907 0 +940 15528.22304461984 4321.805704616348 0 +941 16822.39374421816 820.0898159128985 0 +942 3177.606255781716 820.0898159136095 0 +943 12849.47890487476 2152.656384913164 0 +944 13073.07647653772 2873.849737202373 0 +945 8327.666573283685 1987.523334238995 0 +946 11004.72091372106 259.5942699961332 0 +947 12271.98514369267 1421.895326507942 0 +948 3900.825578892361 4044.333827086158 0 +949 16099.17442110771 4044.33382708549 0 +950 12068.19990469214 3488.950803456739 0 +951 7928.507498042614 3481.938376038602 0 +952 3010.486934868924 2247.386336570385 0 +953 16989.5130651311 2247.386336569912 0 +954 6838.151418680336 1577.797774538163 0 +955 18557.00267445232 5694.064547999547 0 +956 1442.997325542761 5694.064548001164 0 +957 19723.27441200447 2601.515725904796 0 +958 276.7255879936231 2601.515725922467 0 +959 18468.37780914809 3215.653741927955 0 +960 1531.622190853897 3215.653741928719 0 +961 8987.155988412631 253.0463440467311 0 +962 12174.78551377152 1951.42363196735 0 +963 5970.717200427669 2882.080459144843 0 +964 9263.960998122369 2378.521924135534 0 +965 1523.902059294272 312.5888687920592 0 +966 18476.09794070914 312.5888687917555 0 +967 11402.40785255495 1814.071852963729 0 +968 4702.645542936267 1582.682769337505 0 +969 15297.35445706373 1582.682769337389 0 +970 4806.663749895348 3026.924703799207 0 +971 15193.33625010492 3026.924703799028 0 +972 7196.657489961984 1588.336430533954 0 +973 16090.79219125193 5102.49032531653 0 +974 3909.207808742399 5102.49032531334 0 +975 12422.8813435543 2372.452052129323 0 +976 332.2219791360692 5211.084881611423 0 +977 19667.77802086452 5211.084881611187 0 +978 6179.743811256371 3420.796288262783 0 +979 6951.119066995954 4486.875356782797 0 +980 13048.85978310207 4487.189510274926 0 +981 19667.73531006836 786.7913849807928 0 +982 332.264689931126 786.7913849816533 0 +983 5040.773885836378 2704.939707749377 0 +984 14959.10768307118 2704.379584663671 0 +985 19343.31928739878 1879.386418135744 0 +986 656.6807125990692 1879.386418136 0 +987 13975.82122359721 2896.487424405066 0 +988 3122.101405712247 3495.488918515912 0 +989 16877.89859428755 3495.488918515463 0 +990 17550.23301649495 5293.771450255012 0 +991 2449.766983501268 5293.771450256363 0 +992 2601.904900101683 3116.75534054166 0 +993 17398.09509989915 3116.755340541292 0 +994 9854.92211557279 3843.432754088741 0 +995 9064.912616781914 3222.382740563245 0 +996 10935.08738321809 3222.382740563238 0 +997 9227.967907886936 2005.984612329801 0 +998 7832.174740193248 2396.59223701332 0 +999 4120.980538237919 2977.198550107527 0 +1000 15879.01946176226 2977.198550107151 0 +1001 8746.184341746331 2940.048383223566 0 +1002 16964.56363897938 1481.547621640943 0 +1003 3035.436361020852 1481.547621641599 0 +1004 9227.493427253388 1260.236657241599 0 +1005 11249.61195596744 2943.785307954822 0 +1006 11708.51554666859 2486.605357109565 0 +1007 8297.644873216015 5353.949007447971 0 +1008 11702.35512678228 5353.949007448201 0 +1009 2064.486231329743 4426.663145388702 0 +1010 17935.51376867097 4426.663145387831 0 +1011 8745.561513679624 3230.084451460949 0 +1012 11254.43848632041 3230.084451460946 0 +1013 9460.12096169406 2181.492789709943 0 +1014 8883.768487482916 3684.836038992784 0 +1015 11116.23151251714 3684.83603899234 0 +1016 5853.818109289493 2515.487280452197 0 +1017 7054.581850119715 3138.606972107409 0 +1018 16960.78990409959 5211.154137742412 0 +1019 3039.210095887649 5211.154137749743 0 +1020 9748.852319142819 4023.430309150928 0 +1021 11805.07789305487 2130.457724181085 0 +1022 4239.28123307935 258.5411356248724 0 +1023 15754.53044297725 247.4878880068941 0 +1024 14148.15394091045 2513.886901467319 0 +1025 13463.93611963154 5749.831907638224 0 +1026 6536.063880348752 5749.831907636035 0 +1027 9520.427298348775 1228.493116024872 0 +1028 3399.342363028564 2773.159070791506 0 +1029 16600.65763697155 2773.15907079084 0 +1030 4068.113559045606 5705.347761857873 0 +1031 15931.88644094477 5705.347761858098 0 +1032 4473.680733810987 746.9857305681354 0 +1033 15503.89983923185 769.5414312785504 0 +1034 1404.743232497872 5215.757976928063 0 +1035 18595.25676750253 5215.757976926906 0 +1036 10560 2391.060539707571 0 +1037 12422.24868231945 1913.233324292158 0 +1038 6963.722043204627 3462.934163506155 0 +1039 7959.587275266409 2174.692105166522 0 +1040 8341.127949325542 4836.879478712816 0 +1041 11658.87205067397 4836.879478713174 0 +1042 8696.827553347841 4063.62862804308 0 +1043 11303.17244665233 4063.628628042111 0 +1044 19168.72442326376 2418.433220934314 0 +1045 831.2755767329833 2418.433220935379 0 +1046 10479.34908570455 1829.448147495638 0 +1047 12201.62900694961 715.4049161735499 0 +1048 3757.418730635372 3445.574773588773 0 +1049 16242.5812693644 3445.574773587969 0 +1050 11473.03769578883 1488.136967983762 0 +1051 3578.130233112411 3093.369950741863 0 +1052 16421.86976688746 3093.369950741129 0 +1053 19360.46761681194 4098.479605577285 0 +1054 639.5323831850442 4098.479605589328 0 +1055 13100.09171847454 829.4324010839795 0 +1056 7017.086064966164 4757.685438130397 0 +1057 12982.91301744943 4757.699067585677 0 +1058 769.8749373467082 4788.830240707563 0 +1059 19230.1250626543 4788.830240706163 0 +1060 12590.16249154662 1094.933690457482 0 +1061 7097.928829713925 2452.565476033734 0 +1062 10549.11571897475 978.5922417069644 0 +1063 1292.951482936262 4657.126703452848 0 +1064 18707.04851706501 4657.126703452687 0 +1065 12602.18350858968 5733.203189687266 0 +1066 7397.816491381887 5733.203189689817 0 +1067 11742.07958624386 1892.676174257003 0 +1068 5005.6641951402 3956.994886869682 0 +1069 14994.3358048602 3956.994886869658 0 +1070 6574.815835503554 1831.398327916953 0 +1071 8558.320774748336 2949.816017273975 0 +1072 11442.02076260245 2945.874801152608 0 +1073 5257.539119979856 5272.925226527108 0 +1074 14742.4608800213 5272.925226525395 0 +1075 6936.396864266153 5336.410892567456 0 +1076 13063.60313572852 5336.41089256842 0 +1077 7335.627027272714 4804.675511335779 0 +1078 12665.42191643007 4803.345679811086 0 +1079 6106.137957438712 2286.384647955211 0 +1080 10860.56556709352 2378.405994445051 0 +1081 9964.67919731558 1320.067397500271 0 +1082 12556.29346632039 262.8386515832176 0 +1083 4138.375397346459 1620.087355457574 0 +1084 15861.62460265366 1620.087355457449 0 +1085 12580.58467519425 3228.301946781965 0 +1086 17372.67841858258 2514.705039326585 0 +1087 2627.321581419767 2514.705039327256 0 +1088 14193.67233616189 262.981753467087 0 +1089 10751.01205049785 1293.393445908154 0 +1090 8400.698410344623 1142.109047846414 0 +1091 5594.288745362511 3113.734285313193 0 +1092 14393.86382441799 3122.213133382804 0 +1093 13465.12846639955 1902.691699871232 0 +1094 6334.481745915598 5194.098530860779 0 +1095 13665.51825408339 5194.098530862126 0 +1096 8185.056181275717 712.4964805796855 0 +1097 8838.484789570241 2435.662229166789 0 +1098 5649.26936126675 1599.043731629146 0 +1099 14368.04244309951 1587.18062026078 0 +1100 12536.73996574985 805.4875427651918 0 +1101 4145.083901605938 5294.871416346509 0 +1102 15854.91609838575 5294.8714163492 0 +1103 2958.649070135476 2979.575819774304 0 +1104 17041.35092986479 2979.575819773506 0 +1105 10154.30276396277 677.789374409851 0 +1106 9274.19327046278 3622.866173764572 0 +1107 10726.68271181842 3623.451390444025 0 +1108 19208.73190018876 1057.661362284721 0 +1109 791.268099810093 1057.661362284718 0 +1110 5393.148394053178 280.6225668491634 0 +1111 13236.08117589853 1380.274033114138 0 +1112 11377.8864226842 2099.166780643514 0 +1113 14606.26573179628 5734.555533065621 0 +1114 5393.734268198596 5734.555533059276 0 +1115 5448.460899549421 4644.121371014772 0 +1116 14551.53910045094 4644.121371012686 0 +1117 5393.447122291137 3655.428854777738 0 +1118 14606.55287770891 3655.428854777742 0 +1119 7566.25002864825 4059.846053036078 0 +1120 12434.19556609553 4060.246662665049 0 +1121 18272.20618531853 1002.085014646011 0 +1122 1727.793814681714 1002.085014646606 0 +1123 4455.799180633642 2517.00214997597 0 +1124 15544.20081936656 2517.002149975619 0 +1125 13344.71337835 717.2299704534182 0 +1126 12577.95672816579 2601.652795618083 0 +1127 9823.638523163598 629.3359635571753 0 +1128 2721.321210256825 5113.356637709699 0 +1129 17278.67878974069 5113.356637706187 0 +1130 12566.56479629449 2149.129116671431 0 +1131 17131.33496318851 5716.406423685495 0 +1132 2868.665036802389 5716.406423687802 0 +1133 8323.711674434217 3846.849360563745 0 +1134 11676.28832556499 3846.849360562987 0 +1135 13742.87728488811 1511.550624394833 0 +1136 3785.520323002328 1634.622698838609 0 +1137 16214.479676998 1634.622698838175 0 +1138 7907.324926812381 3036.701897776702 0 +1139 9656.476465526017 1930.938516338081 0 +1140 1538.366413036622 4158.432972832468 0 +1141 18461.63358696379 4158.432972832598 0 +1142 11354.43482059414 2668.677902168253 0 +1143 10080 3626.204086449672 0 +1144 7053.011579550853 1056.609906136453 0 +1145 10707.15864464804 261.9337084435714 0 +1146 13007.32328018499 2383.30860901853 0 +1147 8356.559083977991 3271.691945582229 0 +1148 11644.28873516723 3269.728978918805 0 +1149 6590.135465902504 4481.06654344325 0 +1150 13408.89035779222 4481.147186835818 0 +1151 13557.78621298047 1260.725367286174 0 +1152 12091.57529542529 3039.586595777831 0 +1153 2640.832816598497 4853.515879188985 0 +1154 17359.16718340222 4853.515879185722 0 +1155 13089.23503298356 282.1416308734047 0 +1156 10821.80934307766 4246.156637640419 0 +1157 9178.190656916504 4246.156637641572 0 +1158 1173.240392747561 4086.454568082402 0 +1159 18826.75960725189 4086.454568082408 0 +1160 6341.182938305424 2799.88325268084 0 +1161 17862.79559293826 1900.206417464836 0 +1162 2137.204407060267 1900.20641746482 0 +1163 17166.93357557294 1876.510464438481 0 +1164 2833.066424427127 1876.510464438777 0 +1165 3600.853512423304 3747.605012902502 0 +1166 16399.14648757678 3747.605012901602 0 +1167 11519.74063702704 2431.238141304438 0 +1168 7385.69901778317 224.0088719148265 0 +1169 8715.785045065013 5600.196679942932 0 +1170 11284.21495490338 5600.196679923023 0 +1171 17788.74388573489 255.3076818270907 0 +1172 2211.256114271016 255.3076818270923 0 +1173 4180.014318746846 1311.502956578546 0 +1174 15819.98568125317 1311.502956578583 0 +1175 13405.42176034513 3039.846949628762 0 +1176 2170.977304662732 5779.756169255412 0 +1177 17829.02269532935 5779.756169256813 0 +1178 8554.477207541986 3733.992501418597 0 +1179 11445.53053244434 3733.994545688598 0 +1180 4744.222929883024 4453.610778001144 0 +1181 15255.77707011745 4453.610778000813 0 +1182 10241.28335902661 3961.58020340185 0 +1183 8747.657545040705 623.07550522342 0 +1184 6560.080921828031 3055.816849320856 0 +1185 3630.516128548483 1329.360159090155 0 +1186 16369.48387145183 1329.360159089443 0 +1187 18015.3518107437 1571.824875074058 0 +1188 1984.648189256385 1571.824875074748 0 +1189 12051.88248131204 2172.391322821968 0 +1190 13968.00031693088 614.2774914635318 0 +1191 3580.366643365491 4647.103294676094 0 +1192 16419.63335663471 4647.103294673503 0 +1193 6839.700999486936 2935.240091448861 0 +1194 18836.42233385641 1939.784657114073 0 +1195 1163.577666143477 1939.784657115463 0 +1196 5958.316015653157 4494.062989550898 0 +1197 14041.683984346 4494.062989545256 0 +1198 12871.02201529754 2614.387303029627 0 +1199 3723.614680019426 2196.756861421832 0 +1200 16276.3853199806 2196.756861421238 0 +1201 19366.21114346748 244.7489633154714 0 +1202 633.7888565323557 244.748963314421 0 +1203 12957.28791915093 1846.961151440984 0 +1204 8524.279658346111 3523.666099864294 0 +1205 11475.81265389088 3523.690481197151 0 +1206 6035.018341714893 960.2388314239909 0 +1207 2824.075201953408 930.1991649772438 0 +1208 17175.92479804642 930.1991649770011 0 +1209 7102.787550478872 3963.347231974241 0 +1210 9091.952424536485 2429.257931481507 0 +1211 3659.626450838331 5274.692551384108 0 +1212 16340.3735491524 5274.692551388841 0 +1213 17788.77840893136 1272.282936301084 0 +1214 2211.221591068828 1272.282936301773 0 +1215 5606.478712194012 4102.835557110399 0 +1216 14393.29418340324 4102.821345744493 0 +1217 5428.943708300088 820.1929457014199 0 +1218 10576.30415844194 3884.185218198257 0 +1219 9431.407134703091 3881.492163859983 0 +1220 4921.494054294471 5777.521832996042 0 +1221 15078.50594570198 5777.521832989552 0 +1222 3784.189852663014 5760.989350245639 0 +1223 16215.81014732775 5760.989350250654 0 +1224 10802.55855736391 1666.675142781219 0 +1225 10178.60459253531 901.8738402953268 0 +1226 9797.62725457174 1508.840259706403 0 +1227 620.6777376552245 5754.506001537222 0 +1228 19379.32226234462 5754.50600153829 0 +1229 6258.027374858738 1432.948278346884 0 +1230 12910.07205173839 3962.953074892874 0 +1231 8618.514660046632 1347.530497988028 0 +1232 11357.84231130465 1203.445592347807 0 +1233 3255.870550382524 3838.624119952454 0 +1234 16744.12944961787 3838.624119951414 0 +1235 8247.013816899012 2864.033475394691 0 +1236 3317.764795447539 4190.213929596048 0 +1237 16682.2352045533 4190.213929594635 0 +1238 12763.79628270366 1589.870946835771 0 +1239 8646.11512762514 1766.964421191112 0 +1240 12612.69323401314 3019.032520772175 0 +1241 10108.15532798343 2177.932087733228 0 +1242 10720 2407.681107115304 0 +1243 8637.330145874716 2215.534536939525 0 +1244 6637.967199087489 3620.033549565489 0 +1245 9070.891804590607 3489.779043581711 0 +1246 10929.10819540939 3489.779043581731 0 +1247 11755.14814754039 2860.461491788213 0 +1248 7227.889157038091 5257.656926369465 0 +1249 12772.11084295585 5257.656926367899 0 +1250 2441.768635447864 2861.333494324504 0 +1251 17558.23136455382 2861.333494324279 0 +1252 7600.746707396263 3847.920526040195 0 +1253 12397.64555260006 3844.963694257347 0 +1254 11300.29528320916 256.6579944640316 0 +1255 5211.042397214003 586.7217806115857 0 +1256 8837.009029726349 2268.883447473842 0 +1257 7573.27054865132 619.6122664667249 0 +1258 18848.9162618701 2256.069626513796 0 +1259 1151.083738129308 2256.069626514748 0 +1260 17254.52141679086 1252.152891940092 0 +1261 2745.478583209208 1252.152891940693 0 +1262 700.3876545788792 5085.149977878041 0 +1263 19299.61234542231 5085.149977877805 0 +1264 10205.25544109706 2050.503417803538 0 +1265 592.2919223801125 3546.676246627372 0 +1266 19407.70807761255 3546.676246619003 0 +1267 9155.035661697613 3889.954134080448 0 +1268 10844.96433830253 3889.954134078434 0 +1269 5635.535148443477 3501.115732242148 0 +1270 14354.14284400618 3484.240568249847 0 +1271 4515.183881163254 1276.67969558796 0 +1272 15492.71804606972 1266.81842157162 0 +1273 1763.887008908767 5009.41093784331 0 +1274 18236.1129910915 5009.410937841767 0 +1275 19309.5603242361 2932.511649358944 0 +1276 690.4396757657009 2932.511649361962 0 +1277 18733.92794992489 1368.379169392779 0 +1278 1266.072050075019 1368.379169393201 0 +1279 13631.42985180211 2439.829095616463 0 +1280 10327.12433368426 1534.415998740177 0 +1281 4723.944625530471 272.1120254506001 0 +1282 9762.431534167639 3001.251040195057 0 +1283 10263.21428571429 3000 0 +1284 10011.60714285714 2813.17142857143 0 +1285 10017.99660206623 3200.987525694598 0 +1286 9503.856409412067 2781.477606558767 0 +1287 9493.329927372275 3205.882891802091 0 +1288 10501.33526379042 2777.634464725448 0 +1289 10511.62042922317 3209.688326269318 0 +1290 9760 2743.6 0 +1291 9751.302816130639 3257.653505504683 0 +1292 10250.28346644619 2745.105267772203 0 +1293 10266.59083725379 3259.20938827567 0 +1294 10585.17199621975 2999.999999999993 0 +1295 9412.4952557991 3000 0 +1296 9942.696436894941 2971.963640121212 0 +1297 9382.449515283106 2681.684868005113 0 +1298 9382.449515283108 3318.315131994886 0 +1299 9908.376520499782 2668.101533721294 0 +1300 9904.056075653543 3331.12688336324 0 +1301 9603.353848272352 2660.107438991018 0 +1302 9616.544928042369 3351.460958795265 0 +1303 10622.01726505151 2677.396758883887 0 +1304 10622.01726505151 3322.603241116121 0 +1305 9591.784406260869 3045.352805440416 0 +1306 10102.00492343686 2664.527048546767 0 +1307 10107.06467134698 3329.70210436903 0 +1308 10392.98210146005 3350.377535476131 0 +1309 10404.39626643074 2656.725943227556 0 +1310 10412.22544042198 3052.194382397768 0 +1311 10115.75016071536 2955.256821755231 0 +1312 10160.80918445529 3124.925644932945 0 +1313 9853.495256917186 2860.561924807482 0 +1314 9874.602547435728 3137.707405941188 0 +1315 9659.841544820098 2868.916430605442 0 +1316 10355.61750465699 2866.259941083728 0 +1317 9357.211784132336 2854.818981544532 0 +1318 9357.581491558141 3148.782548269064 0 +1319 10644.86055406767 3144.849904571982 0 +1320 10642.48572445349 2853.570059938685 0 +1321 9720.703427374739 3133.168208534395 0 +1322 10297.07838680468 3132.660281172177 0 +1323 9760 2608.663711001643 0 +1324 9758.843402832365 3388.9877036657 0 +1325 10243.74333227692 2610.853558737361 0 +1326 10242.42803051129 3390.674169331803 0 +1327 10143.76733425489 2816.976398079193 0 +1328 10484.14994549207 2924.925322682385 0 +1329 9487.785919914862 3381.477739671092 0 +1330 9487.78591991483 2618.522260328896 0 +1331 10022.37098631351 3078.168207689035 0 +1332 10518.15159564748 3384.997495586968 0 +1333 10518.15159564751 2615.002504413031 0 +1334 9505.037880084894 2910.113164829831 0 +1335 10379.25996824677 3213.63219049109 0 +1336 9627.527490370565 3214.702593325595 0 +1337 10233.64341336169 2884.317583966484 0 +1338 9297.332725916205 3400.02724978711 0 +1339 9297.332725916216 2599.972750212899 0 +1340 9288.749019875029 2754.530520210164 0 +1341 9288.749019875055 3245.469479789797 0 +1342 10710.08076687081 2756.424092503021 0 +1343 10710.08076687084 3243.575907496976 0 +1344 10706.44571462032 2596.345447297821 0 +1345 10706.44571462036 3403.654552702211 0 +$EndNodes +$Elements +2688 +1 1 2 3 100 3 9 +2 1 2 3 100 9 10 +3 1 2 3 100 10 11 +4 1 2 3 100 11 12 +5 1 2 3 100 12 13 +6 1 2 3 100 13 14 +7 1 2 3 100 14 15 +8 1 2 3 100 15 16 +9 1 2 3 100 16 17 +10 1 2 3 100 17 18 +11 1 2 3 100 18 19 +12 1 2 3 100 19 20 +13 1 2 3 100 20 21 +14 1 2 3 100 21 22 +15 1 2 3 100 22 23 +16 1 2 3 100 23 24 +17 1 2 3 100 24 25 +18 1 2 3 100 25 26 +19 1 2 3 100 26 27 +20 1 2 3 100 27 28 +21 1 2 3 100 28 29 +22 1 2 3 100 29 30 +23 1 2 3 100 30 31 +24 1 2 3 100 31 32 +25 1 2 3 100 32 33 +26 1 2 3 100 33 34 +27 1 2 3 100 34 35 +28 1 2 3 100 35 36 +29 1 2 3 100 36 37 +30 1 2 3 100 37 38 +31 1 2 3 100 38 39 +32 1 2 3 100 39 40 +33 1 2 3 100 40 41 +34 1 2 3 100 41 42 +35 1 2 3 100 42 43 +36 1 2 3 100 43 44 +37 1 2 3 100 44 45 +38 1 2 3 100 45 46 +39 1 2 3 100 46 47 +40 1 2 3 100 47 48 +41 1 2 3 100 48 49 +42 1 2 3 100 49 50 +43 1 2 3 100 50 51 +44 1 2 3 100 51 52 +45 1 2 3 100 52 53 +46 1 2 3 100 53 54 +47 1 2 3 100 54 55 +48 1 2 3 100 55 56 +49 1 2 3 100 56 57 +50 1 2 3 100 57 58 +51 1 2 3 100 58 59 +52 1 2 3 100 59 60 +53 1 2 3 100 60 61 +54 1 2 3 100 61 62 +55 1 2 3 100 62 63 +56 1 2 3 100 63 4 +57 1 2 1 101 3 64 +58 1 2 1 101 64 65 +59 1 2 1 101 65 66 +60 1 2 1 101 66 67 +61 1 2 1 101 67 68 +62 1 2 1 101 68 69 +63 1 2 1 101 69 70 +64 1 2 1 101 70 71 +65 1 2 1 101 71 72 +66 1 2 1 101 72 73 +67 1 2 1 101 73 74 +68 1 2 1 101 74 75 +69 1 2 1 101 75 76 +70 1 2 1 101 76 77 +71 1 2 1 101 77 1 +72 1 2 3 102 1 78 +73 1 2 3 102 78 79 +74 1 2 3 102 79 80 +75 1 2 3 102 80 81 +76 1 2 3 102 81 82 +77 1 2 3 102 82 83 +78 1 2 3 102 83 84 +79 1 2 3 102 84 85 +80 1 2 3 102 85 86 +81 1 2 3 102 86 87 +82 1 2 3 102 87 88 +83 1 2 3 102 88 89 +84 1 2 3 102 89 90 +85 1 2 3 102 90 91 +86 1 2 3 102 91 92 +87 1 2 3 102 92 93 +88 1 2 3 102 93 94 +89 1 2 3 102 94 95 +90 1 2 3 102 95 96 +91 1 2 3 102 96 97 +92 1 2 3 102 97 98 +93 1 2 3 102 98 99 +94 1 2 3 102 99 100 +95 1 2 3 102 100 101 +96 1 2 3 102 101 102 +97 1 2 3 102 102 103 +98 1 2 3 102 103 104 +99 1 2 3 102 104 105 +100 1 2 3 102 105 106 +101 1 2 3 102 106 107 +102 1 2 3 102 107 108 +103 1 2 3 102 108 109 +104 1 2 3 102 109 110 +105 1 2 3 102 110 111 +106 1 2 3 102 111 112 +107 1 2 3 102 112 113 +108 1 2 3 102 113 114 +109 1 2 3 102 114 115 +110 1 2 3 102 115 116 +111 1 2 3 102 116 117 +112 1 2 3 102 117 118 +113 1 2 3 102 118 119 +114 1 2 3 102 119 120 +115 1 2 3 102 120 121 +116 1 2 3 102 121 122 +117 1 2 3 102 122 123 +118 1 2 3 102 123 124 +119 1 2 3 102 124 125 +120 1 2 3 102 125 126 +121 1 2 3 102 126 2 +122 1 2 2 103 2 127 +123 1 2 2 103 127 128 +124 1 2 2 103 128 129 +125 1 2 2 103 129 130 +126 1 2 2 103 130 131 +127 1 2 2 103 131 132 +128 1 2 2 103 132 133 +129 1 2 2 103 133 134 +130 1 2 2 103 134 135 +131 1 2 2 103 135 136 +132 1 2 2 103 136 137 +133 1 2 2 103 137 138 +134 1 2 2 103 138 139 +135 1 2 2 103 139 140 +136 1 2 2 103 140 4 +137 2 2 1 111 605 248 1123 +138 2 2 1 111 1124 249 606 +139 2 2 1 111 478 689 1042 +140 2 2 1 111 479 1043 690 +141 2 2 1 111 510 605 1123 +142 2 2 1 111 511 1124 606 +143 2 2 1 111 796 1007 224 +144 2 2 1 111 1008 797 225 +145 2 2 1 111 292 439 966 +146 2 2 1 111 293 965 440 +147 2 2 1 111 197 730 451 +148 2 2 1 111 198 452 731 +149 2 2 1 111 142 143 516 +150 2 2 1 111 453 1149 538 +151 2 2 1 111 454 539 1150 +152 2 2 1 111 159 564 160 +153 2 2 1 111 164 165 565 +154 2 2 1 111 385 593 771 +155 2 2 1 111 542 858 927 +156 2 2 1 111 543 928 857 +157 2 2 1 111 327 784 558 +158 2 2 1 111 328 559 785 +159 2 2 1 111 204 563 765 +160 2 2 1 111 294 643 490 +161 2 2 1 111 628 893 885 +162 2 2 1 111 300 443 726 +163 2 2 1 111 299 727 444 +164 2 2 1 111 115 547 116 +165 2 2 1 111 764 639 209 +166 2 2 1 111 763 208 640 +167 2 2 1 111 444 639 764 +168 2 2 1 111 443 763 640 +169 2 2 1 111 304 904 729 +170 2 2 1 111 304 562 904 +171 2 2 1 111 270 445 666 +172 2 2 1 111 11 437 753 +173 2 2 1 111 61 754 438 +174 2 2 1 111 371 545 1112 +175 2 2 1 111 181 487 627 +176 2 2 1 111 292 668 449 +177 2 2 1 111 293 450 669 +178 2 2 1 111 558 784 1101 +179 2 2 1 111 559 1102 785 +180 2 2 1 111 404 589 954 +181 2 2 1 111 124 439 737 +182 2 2 1 111 80 738 440 +183 2 2 1 111 355 533 1108 +184 2 2 1 111 356 1109 534 +185 2 2 1 111 281 605 510 +186 2 2 1 111 282 511 606 +187 2 2 1 111 159 624 564 +188 2 2 1 111 312 518 566 +189 2 2 1 111 313 567 519 +190 2 2 1 111 113 1088 562 +191 2 2 1 111 1140 186 809 +192 2 2 1 111 1141 810 187 +193 2 2 1 111 233 460 671 +194 2 2 1 111 234 672 461 +195 2 2 1 111 353 709 553 +196 2 2 1 111 354 554 710 +197 2 2 1 111 348 628 885 +198 2 2 1 111 288 671 460 +199 2 2 1 111 289 461 672 +200 2 2 1 111 355 1108 905 +201 2 2 1 111 356 906 1109 +202 2 2 1 111 193 538 1149 +203 2 2 1 111 194 1150 539 +204 2 2 1 111 297 674 456 +205 2 2 1 111 298 457 673 +206 2 2 1 111 180 626 492 +207 2 2 1 111 1011 212 485 +208 2 2 1 111 1012 486 213 +209 2 2 1 111 142 516 619 +210 2 2 1 111 11 956 437 +211 2 2 1 111 61 438 955 +212 2 2 1 111 689 195 1042 +213 2 2 1 111 1043 196 690 +214 2 2 1 111 180 614 560 +215 2 2 1 111 181 561 615 +216 2 2 1 111 331 542 550 +217 2 2 1 111 332 551 543 +218 2 2 1 111 5 624 159 +219 2 2 1 111 491 594 987 +220 2 2 1 111 260 550 542 +221 2 2 1 111 261 543 551 +222 2 2 1 111 401 1244 1038 +223 2 2 1 111 268 1149 979 +224 2 2 1 111 269 980 1150 +225 2 2 1 111 233 1032 460 +226 2 2 1 111 234 461 1033 +227 2 2 1 111 512 843 1206 +228 2 2 1 111 292 790 439 +229 2 2 1 111 293 440 791 +230 2 2 1 111 147 148 590 +231 2 2 1 111 273 1097 816 +232 2 2 1 111 259 497 1096 +233 2 2 1 111 87 720 460 +234 2 2 1 111 117 461 719 +235 2 2 1 111 192 947 500 +236 2 2 1 111 43 44 499 +237 2 2 1 111 28 29 498 +238 2 2 1 111 124 966 439 +239 2 2 1 111 80 440 965 +240 2 2 1 111 259 1096 496 +241 2 2 1 111 498 1007 796 +242 2 2 1 111 499 797 1008 +243 2 2 1 111 103 506 645 +244 2 2 1 111 193 566 518 +245 2 2 1 111 194 519 567 +246 2 2 1 111 594 170 987 +247 2 2 1 111 113 562 114 +248 2 2 1 111 115 756 547 +249 2 2 1 111 313 617 441 +250 2 2 1 111 179 771 593 +251 2 2 1 111 275 484 1151 +252 2 2 1 111 180 1204 485 +253 2 2 1 111 181 486 1205 +254 2 2 1 111 209 858 542 +255 2 2 1 111 208 543 857 +256 2 2 1 111 270 647 445 +257 2 2 1 111 143 625 516 +258 2 2 1 111 301 1206 843 +259 2 2 1 111 550 1140 809 +260 2 2 1 111 551 810 1141 +261 2 2 1 111 275 937 1125 +262 2 2 1 111 214 517 972 +263 2 2 1 111 192 500 692 +264 2 2 1 111 300 763 443 +265 2 2 1 111 299 444 764 +266 2 2 1 111 654 822 338 +267 2 2 1 111 655 340 823 +268 2 2 1 111 469 822 654 +269 2 2 1 111 470 655 823 +270 2 2 1 111 385 765 563 +271 2 2 1 111 268 566 1149 +272 2 2 1 111 269 1150 567 +273 2 2 1 111 305 581 652 +274 2 2 1 111 306 653 582 +275 2 2 1 111 478 1133 689 +276 2 2 1 111 479 690 1134 +277 2 2 1 111 268 979 761 +278 2 2 1 111 269 762 980 +279 2 2 1 111 468 1038 1244 +280 2 2 1 111 384 888 677 +281 2 2 1 111 38 801 403 +282 2 2 1 111 34 402 800 +283 2 2 1 111 242 677 607 +284 2 2 1 111 250 1055 623 +285 2 2 1 111 297 456 742 +286 2 2 1 111 298 743 457 +287 2 2 1 111 171 988 503 +288 2 2 1 111 172 504 989 +289 2 2 1 111 241 450 734 +290 2 2 1 111 240 733 449 +291 2 2 1 111 39 782 801 +292 2 2 1 111 33 800 781 +293 2 2 1 111 292 449 733 +294 2 2 1 111 293 734 450 +295 2 2 1 111 324 498 796 +296 2 2 1 111 325 797 499 +297 2 2 1 111 370 500 947 +298 2 2 1 111 160 564 860 +299 2 2 1 111 165 861 565 +300 2 2 1 111 458 948 1165 +301 2 2 1 111 459 1166 949 +302 2 2 1 111 97 687 496 +303 2 2 1 111 259 496 1257 +304 2 2 1 111 309 579 521 +305 2 2 1 111 310 522 580 +306 2 2 1 111 275 1125 484 +307 2 2 1 111 257 502 1087 +308 2 2 1 111 258 1086 501 +309 2 2 1 111 199 521 579 +310 2 2 1 111 200 580 522 +311 2 2 1 111 16 698 662 +312 2 2 1 111 56 663 699 +313 2 2 1 111 16 17 698 +314 2 2 1 111 55 56 699 +315 2 2 1 111 275 618 937 +316 2 2 1 111 101 667 629 +317 2 2 1 111 242 607 1039 +318 2 2 1 111 87 460 1022 +319 2 2 1 111 117 1023 461 +320 2 2 1 111 458 585 948 +321 2 2 1 111 459 949 586 +322 2 2 1 111 305 652 535 +323 2 2 1 111 306 536 653 +324 2 2 1 111 309 540 1199 +325 2 2 1 111 310 1200 541 +326 2 2 1 111 203 804 570 +327 2 2 1 111 241 911 450 +328 2 2 1 111 240 449 912 +329 2 2 1 111 255 535 652 +330 2 2 1 111 256 653 536 +331 2 2 1 111 290 714 466 +332 2 2 1 111 291 467 715 +333 2 2 1 111 512 589 843 +334 2 2 1 111 484 1055 851 +335 2 2 1 111 178 677 888 +336 2 2 1 111 197 451 746 +337 2 2 1 111 198 745 452 +338 2 2 1 111 75 534 982 +339 2 2 1 111 129 981 533 +340 2 2 1 111 138 530 977 +341 2 2 1 111 66 976 529 +342 2 2 1 111 190 954 589 +343 2 2 1 111 72 532 73 +344 2 2 1 111 131 531 132 +345 2 2 1 111 259 647 517 +346 2 2 1 111 284 667 703 +347 2 2 1 111 540 605 1199 +348 2 2 1 111 541 1200 606 +349 2 2 1 111 180 485 1147 +350 2 2 1 111 181 1148 486 +351 2 2 1 111 102 506 103 +352 2 2 1 111 192 609 947 +353 2 2 1 111 540 1028 1051 +354 2 2 1 111 541 1052 1029 +355 2 2 1 111 169 963 493 +356 2 2 1 111 93 520 94 +357 2 2 1 111 300 501 595 +358 2 2 1 111 299 596 502 +359 2 2 1 111 148 1036 590 +360 2 2 1 111 581 999 652 +361 2 2 1 111 582 653 1000 +362 2 2 1 111 427 947 609 +363 2 2 1 111 359 744 831 +364 2 2 1 111 135 523 136 +365 2 2 1 111 68 524 69 +366 2 2 1 111 289 719 461 +367 2 2 1 111 288 460 720 +368 2 2 1 111 401 657 1244 +369 2 2 1 111 294 490 859 +370 2 2 1 111 171 527 988 +371 2 2 1 111 172 989 528 +372 2 2 1 111 485 212 787 +373 2 2 1 111 486 788 213 +374 2 2 1 111 411 562 1088 +375 2 2 1 111 513 1112 545 +376 2 2 1 111 290 466 759 +377 2 2 1 111 291 760 467 +378 2 2 1 111 404 972 517 +379 2 2 1 111 914 680 182 +380 2 2 1 111 915 183 681 +381 2 2 1 111 239 1004 507 +382 2 2 1 111 510 680 914 +383 2 2 1 111 511 915 681 +384 2 2 1 111 314 1139 490 +385 2 2 1 111 22 759 1114 +386 2 2 1 111 50 1113 760 +387 2 2 1 111 274 442 670 +388 2 2 1 111 270 666 520 +389 2 2 1 111 215 896 482 +390 2 2 1 111 169 493 978 +391 2 2 1 111 230 1232 446 +392 2 2 1 111 405 874 616 +393 2 2 1 111 34 697 402 +394 2 2 1 111 38 403 696 +395 2 2 1 111 424 660 886 +396 2 2 1 111 425 887 661 +397 2 2 1 111 313 441 924 +398 2 2 1 111 284 629 667 +399 2 2 1 111 195 689 603 +400 2 2 1 111 196 604 690 +401 2 2 1 111 331 550 809 +402 2 2 1 111 332 810 551 +403 2 2 1 111 386 540 1051 +404 2 2 1 111 387 1052 541 +405 2 2 1 111 275 1151 555 +406 2 2 1 111 250 623 1100 +407 2 2 1 111 204 490 643 +408 2 2 1 111 514 816 1097 +409 2 2 1 111 141 142 619 +410 2 2 1 111 312 751 518 +411 2 2 1 111 313 519 752 +412 2 2 1 111 295 614 610 +413 2 2 1 111 296 611 615 +414 2 2 1 111 497 1090 277 +415 2 2 1 111 250 851 1055 +416 2 2 1 111 305 970 581 +417 2 2 1 111 306 582 971 +418 2 2 1 111 192 446 711 +419 2 2 1 111 230 695 515 +420 2 2 1 111 178 917 677 +421 2 2 1 111 421 1039 607 +422 2 2 1 111 370 1060 500 +423 2 2 1 111 171 503 1051 +424 2 2 1 111 172 1052 504 +425 2 2 1 111 329 927 858 +426 2 2 1 111 330 857 928 +427 2 2 1 111 327 1073 819 +428 2 2 1 111 328 820 1074 +429 2 2 1 111 39 40 782 +430 2 2 1 111 32 33 781 +431 2 2 1 111 704 1098 865 +432 2 2 1 111 1211 473 285 +433 2 2 1 111 1212 286 474 +434 2 2 1 111 170 494 987 +435 2 2 1 111 311 574 1061 +436 2 2 1 111 207 616 874 +437 2 2 1 111 327 597 1073 +438 2 2 1 111 328 1074 598 +439 2 2 1 111 145 146 552 +440 2 2 1 111 321 678 730 +441 2 2 1 111 322 731 679 +442 2 2 1 111 491 1024 893 +443 2 2 1 111 305 721 970 +444 2 2 1 111 306 971 722 +445 2 2 1 111 269 1230 762 +446 2 2 1 111 285 473 746 +447 2 2 1 111 286 745 474 +448 2 2 1 111 210 721 587 +449 2 2 1 111 211 588 722 +450 2 2 1 111 217 704 865 +451 2 2 1 111 392 587 721 +452 2 2 1 111 393 722 588 +453 2 2 1 111 268 761 1209 +454 2 2 1 111 678 1153 730 +455 2 2 1 111 679 731 1154 +456 2 2 1 111 297 1034 956 +457 2 2 1 111 298 955 1035 +458 2 2 1 111 231 886 660 +459 2 2 1 111 232 661 887 +460 2 2 1 111 75 982 76 +461 2 2 1 111 128 981 129 +462 2 2 1 111 507 1004 1027 +463 2 2 1 111 460 1032 546 +464 2 2 1 111 461 547 1033 +465 2 2 1 111 31 781 1169 +466 2 2 1 111 41 1170 782 +467 2 2 1 111 138 977 139 +468 2 2 1 111 65 976 66 +469 2 2 1 111 307 1080 442 +470 2 2 1 111 23 24 633 +471 2 2 1 111 48 49 632 +472 2 2 1 111 181 950 487 +473 2 2 1 111 97 496 913 +474 2 2 1 111 197 568 730 +475 2 2 1 111 198 731 569 +476 2 2 1 111 214 607 922 +477 2 2 1 111 317 610 614 +478 2 2 1 111 318 615 611 +479 2 2 1 111 376 703 667 +480 2 2 1 111 578 1086 953 +481 2 2 1 111 577 952 1087 +482 2 2 1 111 493 963 1091 +483 2 2 1 111 264 944 594 +484 2 2 1 111 334 603 689 +485 2 2 1 111 335 690 604 +486 2 2 1 111 259 622 497 +487 2 2 1 111 180 560 1204 +488 2 2 1 111 181 1205 561 +489 2 2 1 111 253 518 751 +490 2 2 1 111 254 752 519 +491 2 2 1 111 248 540 999 +492 2 2 1 111 249 1000 541 +493 2 2 1 111 164 565 670 +494 2 2 1 111 386 999 540 +495 2 2 1 111 387 541 1000 +496 2 2 1 111 397 922 607 +497 2 2 1 111 369 1027 1004 +498 2 2 1 111 594 944 1175 +499 2 2 1 111 1182 316 708 +500 2 2 1 111 502 596 1087 +501 2 2 1 111 501 1086 595 +502 2 2 1 111 295 689 1133 +503 2 2 1 111 296 1134 690 +504 2 2 1 111 216 664 412 +505 2 2 1 111 513 1021 1067 +506 2 2 1 111 201 583 918 +507 2 2 1 111 308 570 718 +508 2 2 1 111 91 1110 608 +509 2 2 1 111 300 726 501 +510 2 2 1 111 299 502 727 +511 2 2 1 111 1096 497 277 +512 2 2 1 111 553 709 1042 +513 2 2 1 111 554 1043 710 +514 2 2 1 111 143 144 625 +515 2 2 1 111 314 490 778 +516 2 2 1 111 248 605 540 +517 2 2 1 111 249 541 606 +518 2 2 1 111 307 442 631 +519 2 2 1 111 323 477 832 +520 2 2 1 111 180 492 951 +521 2 2 1 111 215 691 883 +522 2 2 1 111 192 692 446 +523 2 2 1 111 397 607 677 +524 2 2 1 111 230 446 695 +525 2 2 1 111 278 583 798 +526 2 2 1 111 496 687 1257 +527 2 2 1 111 258 501 749 +528 2 2 1 111 257 750 502 +529 2 2 1 111 230 515 736 +530 2 2 1 111 170 936 494 +531 2 2 1 111 526 1182 708 +532 2 2 1 111 195 909 553 +533 2 2 1 111 196 554 910 +534 2 2 1 111 435 623 1055 +535 2 2 1 111 23 633 759 +536 2 2 1 111 49 760 632 +537 2 2 1 111 259 517 622 +538 2 2 1 111 202 919 591 +539 2 2 1 111 216 412 739 +540 2 2 1 111 616 967 1050 +541 2 2 1 111 3 9 807 +542 2 2 1 111 4 808 63 +543 2 2 1 111 64 807 976 +544 2 2 1 111 140 977 808 +545 2 2 1 111 491 893 1279 +546 2 2 1 111 271 527 660 +547 2 2 1 111 272 661 528 +548 2 2 1 111 378 728 634 +549 2 2 1 111 379 635 729 +550 2 2 1 111 43 499 815 +551 2 2 1 111 29 814 498 +552 2 2 1 111 246 716 525 +553 2 2 1 111 277 1090 537 +554 2 2 1 111 471 1123 581 +555 2 2 1 111 472 582 1124 +556 2 2 1 111 303 634 728 +557 2 2 1 111 304 729 635 +558 2 2 1 111 85 621 86 +559 2 2 1 111 118 620 119 +560 2 2 1 111 513 545 1021 +561 2 2 1 111 1139 688 265 +562 2 2 1 111 292 966 668 +563 2 2 1 111 293 669 965 +564 2 2 1 111 492 630 951 +565 2 2 1 111 214 922 517 +566 2 2 1 111 3 807 64 +567 2 2 1 111 4 140 808 +568 2 2 1 111 289 672 693 +569 2 2 1 111 288 694 671 +570 2 2 1 111 568 1236 1233 +571 2 2 1 111 569 1234 1237 +572 2 2 1 111 302 515 695 +573 2 2 1 111 248 581 1123 +574 2 2 1 111 249 1124 582 +575 2 2 1 111 323 1093 477 +576 2 2 1 111 324 845 498 +577 2 2 1 111 325 499 844 +578 2 2 1 111 271 568 1233 +579 2 2 1 111 272 1234 569 +580 2 2 1 111 204 765 778 +581 2 2 1 111 258 953 1086 +582 2 2 1 111 257 1087 952 +583 2 2 1 111 369 570 804 +584 2 2 1 111 311 1061 862 +585 2 2 1 111 484 851 1111 +586 2 2 1 111 127 828 981 +587 2 2 1 111 77 982 829 +588 2 2 1 111 31 32 781 +589 2 2 1 111 40 41 782 +590 2 2 1 111 317 951 630 +591 2 2 1 111 457 743 990 +592 2 2 1 111 456 991 742 +593 2 2 1 111 404 1144 589 +594 2 2 1 111 91 608 92 +595 2 2 1 111 278 1138 626 +596 2 2 1 111 152 584 153 +597 2 2 1 111 243 678 742 +598 2 2 1 111 244 743 679 +599 2 2 1 111 281 510 914 +600 2 2 1 111 282 915 511 +601 2 2 1 111 22 23 759 +602 2 2 1 111 49 50 760 +603 2 2 1 111 359 592 744 +604 2 2 1 111 418 742 678 +605 2 2 1 111 419 679 743 +606 2 2 1 111 570 1004 718 +607 2 2 1 111 357 1091 963 +608 2 2 1 111 200 522 1002 +609 2 2 1 111 199 1003 521 +610 2 2 1 111 203 482 997 +611 2 2 1 111 279 789 591 +612 2 2 1 111 210 587 1091 +613 2 2 1 111 211 1092 588 +614 2 2 1 111 453 538 848 +615 2 2 1 111 454 849 539 +616 2 2 1 111 404 517 1144 +617 2 2 1 111 188 599 877 +618 2 2 1 111 189 878 600 +619 2 2 1 111 6 164 670 +620 2 2 1 111 102 629 506 +621 2 2 1 111 398 877 599 +622 2 2 1 111 399 600 878 +623 2 2 1 111 336 939 535 +624 2 2 1 111 337 536 940 +625 2 2 1 111 492 626 1138 +626 2 2 1 111 413 761 979 +627 2 2 1 111 414 980 762 +628 2 2 1 111 2 126 828 +629 2 2 1 111 2 828 127 +630 2 2 1 111 1 829 78 +631 2 2 1 111 1 77 829 +632 2 2 1 111 72 958 532 +633 2 2 1 111 132 531 957 +634 2 2 1 111 441 1230 924 +635 2 2 1 111 210 1091 897 +636 2 2 1 111 211 898 1092 +637 2 2 1 111 201 799 583 +638 2 2 1 111 336 535 948 +639 2 2 1 111 337 949 536 +640 2 2 1 111 672 1186 693 +641 2 2 1 111 671 694 1185 +642 2 2 1 111 260 1265 524 +643 2 2 1 111 261 523 1266 +644 2 2 1 111 304 547 756 +645 2 2 1 111 466 1114 759 +646 2 2 1 111 467 760 1113 +647 2 2 1 111 324 1248 508 +648 2 2 1 111 325 509 1249 +649 2 2 1 111 297 956 674 +650 2 2 1 111 298 673 955 +651 2 2 1 111 94 520 666 +652 2 2 1 111 239 507 703 +653 2 2 1 111 205 630 732 +654 2 2 1 111 284 507 856 +655 2 2 1 111 277 537 818 +656 2 2 1 111 487 1152 627 +657 2 2 1 111 46 842 509 +658 2 2 1 111 26 508 841 +659 2 2 1 111 64 976 65 +660 2 2 1 111 139 977 140 +661 2 2 1 111 560 614 1133 +662 2 2 1 111 561 1134 615 +663 2 2 1 111 274 850 442 +664 2 2 1 111 215 1256 896 +665 2 2 1 111 514 938 816 +666 2 2 1 111 280 827 616 +667 2 2 1 111 171 1051 1028 +668 2 2 1 111 172 1029 1052 +669 2 2 1 111 443 599 726 +670 2 2 1 111 444 727 600 +671 2 2 1 111 324 508 1066 +672 2 2 1 111 325 1065 509 +673 2 2 1 111 477 1146 832 +674 2 2 1 111 279 627 1152 +675 2 2 1 111 494 1092 987 +676 2 2 1 111 190 589 1229 +677 2 2 1 111 127 981 128 +678 2 2 1 111 76 982 77 +679 2 2 1 111 846 1009 321 +680 2 2 1 111 847 322 1010 +681 2 2 1 111 239 537 718 +682 2 2 1 111 28 498 845 +683 2 2 1 111 44 844 499 +684 2 2 1 111 279 641 789 +685 2 2 1 111 1009 678 321 +686 2 2 1 111 1010 322 679 +687 2 2 1 111 402 553 909 +688 2 2 1 111 403 910 554 +689 2 2 1 111 278 798 665 +690 2 2 1 111 337 806 586 +691 2 2 1 111 336 585 805 +692 2 2 1 111 246 525 862 +693 2 2 1 111 342 630 492 +694 2 2 1 111 303 1281 546 +695 2 2 1 111 161 571 162 +696 2 2 1 111 283 544 786 +697 2 2 1 111 216 563 664 +698 2 2 1 111 397 677 917 +699 2 2 1 111 432 608 1110 +700 2 2 1 111 166 167 573 +701 2 2 1 111 244 990 743 +702 2 2 1 111 243 742 991 +703 2 2 1 111 388 1175 944 +704 2 2 1 111 319 603 1040 +705 2 2 1 111 320 1041 604 +706 2 2 1 111 135 777 523 +707 2 2 1 111 69 524 776 +708 2 2 1 111 231 548 886 +709 2 2 1 111 232 887 549 +710 2 2 1 111 184 535 939 +711 2 2 1 111 185 940 536 +712 2 2 1 111 331 886 548 +713 2 2 1 111 332 549 887 +714 2 2 1 111 367 532 958 +715 2 2 1 111 368 957 531 +716 2 2 1 111 302 889 515 +717 2 2 1 111 96 687 97 +718 2 2 1 111 86 621 720 +719 2 2 1 111 118 719 620 +720 2 2 1 111 512 1229 589 +721 2 2 1 111 447 1235 626 +722 2 2 1 111 691 1239 883 +723 2 2 1 111 705 923 1099 +724 2 2 1 111 342 732 630 +725 2 2 1 111 412 1036 1242 +726 2 2 1 111 136 523 675 +727 2 2 1 111 68 676 524 +728 2 2 1 111 216 644 593 +729 2 2 1 111 255 948 535 +730 2 2 1 111 256 536 949 +731 2 2 1 111 218 923 705 +732 2 2 1 111 353 553 1267 +733 2 2 1 111 354 1268 554 +734 2 2 1 111 318 656 950 +735 2 2 1 111 184 747 535 +736 2 2 1 111 185 536 748 +737 2 2 1 111 171 1103 527 +738 2 2 1 111 172 528 1104 +739 2 2 1 111 411 642 904 +740 2 2 1 111 305 535 747 +741 2 2 1 111 306 748 536 +742 2 2 1 111 283 505 765 +743 2 2 1 111 487 950 656 +744 2 2 1 111 228 714 1115 +745 2 2 1 111 229 1116 715 +746 2 2 1 111 365 941 1208 +747 2 2 1 111 366 1207 942 +748 2 2 1 111 270 520 843 +749 2 2 1 111 177 613 856 +750 2 2 1 111 19 907 558 +751 2 2 1 111 53 559 908 +752 2 2 1 111 215 482 691 +753 2 2 1 111 378 1271 968 +754 2 2 1 111 379 969 1272 +755 2 2 1 111 219 902 1034 +756 2 2 1 111 220 1035 903 +757 2 2 1 111 448 627 1247 +758 2 2 1 111 199 579 1136 +759 2 2 1 111 200 1137 580 +760 2 2 1 111 378 634 1271 +761 2 2 1 111 379 1272 635 +762 2 2 1 111 371 1167 545 +763 2 2 1 111 131 683 531 +764 2 2 1 111 73 532 682 +765 2 2 1 111 342 723 732 +766 2 2 1 111 488 1208 941 +767 2 2 1 111 489 942 1207 +768 2 2 1 111 426 1100 623 +769 2 2 1 111 326 487 656 +770 2 2 1 111 26 1066 508 +771 2 2 1 111 46 509 1065 +772 2 2 1 111 280 592 827 +773 2 2 1 111 262 1044 531 +774 2 2 1 111 263 532 1045 +775 2 2 1 111 307 572 1080 +776 2 2 1 111 251 469 654 +777 2 2 1 111 252 655 470 +778 2 2 1 111 640 1044 1258 +779 2 2 1 111 639 1259 1045 +780 2 2 1 111 368 531 1044 +781 2 2 1 111 367 1045 532 +782 2 2 1 111 193 769 538 +783 2 2 1 111 194 539 770 +784 2 2 1 111 301 520 757 +785 2 2 1 111 453 848 1056 +786 2 2 1 111 454 1057 849 +787 2 2 1 111 230 831 744 +788 2 2 1 111 612 688 1139 +789 2 2 1 111 475 1115 714 +790 2 2 1 111 476 715 1116 +791 2 2 1 111 260 927 1265 +792 2 2 1 111 261 1266 928 +793 2 2 1 111 275 555 642 +794 2 2 1 111 223 962 609 +795 2 2 1 111 522 1163 1002 +796 2 2 1 111 521 1003 1164 +797 2 2 1 111 864 1041 320 +798 2 2 1 111 863 319 1040 +799 2 2 1 111 218 904 642 +800 2 2 1 111 284 703 507 +801 2 2 1 111 280 616 1050 +802 2 2 1 111 441 617 929 +803 2 2 1 111 485 654 1011 +804 2 2 1 111 486 1012 655 +805 2 2 1 111 412 1242 572 +806 2 2 1 111 471 983 680 +807 2 2 1 111 472 681 984 +808 2 2 1 111 199 1136 1185 +809 2 2 1 111 200 1186 1137 +810 2 2 1 111 235 1101 784 +811 2 2 1 111 236 785 1102 +812 2 2 1 111 388 929 617 +813 2 2 1 111 303 546 1032 +814 2 2 1 111 304 1033 547 +815 2 2 1 111 100 667 101 +816 2 2 1 111 360 680 983 +817 2 2 1 111 361 984 681 +818 2 2 1 111 390 1086 578 +819 2 2 1 111 389 577 1087 +820 2 2 1 111 253 1215 518 +821 2 2 1 111 254 519 1216 +822 2 2 1 111 326 1152 487 +823 2 2 1 111 192 711 609 +824 2 2 1 111 261 1053 523 +825 2 2 1 111 260 524 1054 +826 2 2 1 111 505 778 765 +827 2 2 1 111 443 640 1258 +828 2 2 1 111 444 1259 639 +829 2 2 1 111 552 873 1241 +830 2 2 1 111 301 843 520 +831 2 2 1 111 343 684 925 +832 2 2 1 111 344 926 685 +833 2 2 1 111 86 720 87 +834 2 2 1 111 117 719 118 +835 2 2 1 111 276 664 1264 +836 2 2 1 111 370 1238 851 +837 2 2 1 111 203 802 482 +838 2 2 1 111 283 771 544 +839 2 2 1 111 273 564 624 +840 2 2 1 111 223 1067 1021 +841 2 2 1 111 93 757 520 +842 2 2 1 111 331 960 542 +843 2 2 1 111 332 543 959 +844 2 2 1 111 593 644 1224 +845 2 2 1 111 684 1068 925 +846 2 2 1 111 685 926 1069 +847 2 2 1 111 339 860 564 +848 2 2 1 111 341 565 861 +849 2 2 1 111 309 521 952 +850 2 2 1 111 310 953 522 +851 2 2 1 111 209 542 960 +852 2 2 1 111 208 959 543 +853 2 2 1 111 264 832 1146 +854 2 2 1 111 342 492 1138 +855 2 2 1 111 202 1142 919 +856 2 2 1 111 255 1165 948 +857 2 2 1 111 256 949 1166 +858 2 2 1 111 239 818 537 +859 2 2 1 111 433 1169 781 +860 2 2 1 111 434 782 1170 +861 2 2 1 111 311 732 723 +862 2 2 1 111 437 956 1034 +863 2 2 1 111 438 1035 955 +864 2 2 1 111 500 1047 692 +865 2 2 1 111 467 1074 715 +866 2 2 1 111 466 714 1073 +867 2 2 1 111 271 660 774 +868 2 2 1 111 272 775 661 +869 2 2 1 111 319 800 909 +870 2 2 1 111 320 910 801 +871 2 2 1 111 358 987 1092 +872 2 2 1 111 583 1235 918 +873 2 2 1 111 271 988 527 +874 2 2 1 111 272 528 989 +875 2 2 1 111 108 713 1082 +876 2 2 1 111 418 779 601 +877 2 2 1 111 419 602 780 +878 2 2 1 111 586 806 1192 +879 2 2 1 111 585 1191 805 +880 2 2 1 111 206 656 853 +881 2 2 1 111 66 529 834 +882 2 2 1 111 138 833 530 +883 2 2 1 111 349 1108 533 +884 2 2 1 111 350 534 1109 +885 2 2 1 111 199 694 1003 +886 2 2 1 111 200 1002 693 +887 2 2 1 111 321 568 774 +888 2 2 1 111 322 775 569 +889 2 2 1 111 271 774 568 +890 2 2 1 111 272 569 775 +891 2 2 1 111 468 921 1209 +892 2 2 1 111 114 562 756 +893 2 2 1 111 489 1003 694 +894 2 2 1 111 488 693 1002 +895 2 2 1 111 351 834 529 +896 2 2 1 111 352 530 833 +897 2 2 1 111 129 533 836 +898 2 2 1 111 75 835 534 +899 2 2 1 111 193 518 1196 +900 2 2 1 111 194 1197 519 +901 2 2 1 111 276 1241 873 +902 2 2 1 111 300 549 763 +903 2 2 1 111 299 764 548 +904 2 2 1 111 351 529 1058 +905 2 2 1 111 352 1059 530 +906 2 2 1 111 355 836 533 +907 2 2 1 111 356 534 835 +908 2 2 1 111 1089 179 827 +909 2 2 1 111 347 1229 865 +910 2 2 1 111 283 786 613 +911 2 2 1 111 230 744 1232 +912 2 2 1 111 349 533 981 +913 2 2 1 111 350 982 534 +914 2 2 1 111 262 531 985 +915 2 2 1 111 263 986 532 +916 2 2 1 111 61 62 754 +917 2 2 1 111 10 11 753 +918 2 2 1 111 274 670 565 +919 2 2 1 111 303 700 1281 +920 2 2 1 111 412 590 1036 +921 2 2 1 111 260 1158 550 +922 2 2 1 111 261 551 1159 +923 2 2 1 111 299 548 894 +924 2 2 1 111 300 895 549 +925 2 2 1 111 103 645 1145 +926 2 2 1 111 309 1028 540 +927 2 2 1 111 310 541 1029 +928 2 2 1 111 79 738 80 +929 2 2 1 111 124 737 125 +930 2 2 1 111 278 626 1235 +931 2 2 1 111 38 39 801 +932 2 2 1 111 33 34 800 +933 2 2 1 111 420 954 1070 +934 2 2 1 111 421 817 875 +935 2 2 1 111 98 638 99 +936 2 2 1 111 563 1264 664 +937 2 2 1 111 441 811 1230 +938 2 2 1 111 538 769 1094 +939 2 2 1 111 539 1095 770 +940 2 2 1 111 517 647 1144 +941 2 2 1 111 203 688 612 +942 2 2 1 111 97 913 98 +943 2 2 1 111 506 1127 1105 +944 2 2 1 111 273 816 854 +945 2 2 1 111 204 778 490 +946 2 2 1 111 591 919 1247 +947 2 2 1 111 382 706 796 +948 2 2 1 111 383 797 707 +949 2 2 1 111 205 852 630 +950 2 2 1 111 217 865 686 +951 2 2 1 111 259 1257 647 +952 2 2 1 111 372 583 799 +953 2 2 1 111 485 1204 654 +954 2 2 1 111 486 655 1205 +955 2 2 1 111 449 668 1171 +956 2 2 1 111 450 1172 669 +957 2 2 1 111 221 879 790 +958 2 2 1 111 222 791 880 +959 2 2 1 111 284 856 1127 +960 2 2 1 111 279 1247 627 +961 2 2 1 111 363 749 556 +962 2 2 1 111 364 557 750 +963 2 2 1 111 501 726 1161 +964 2 2 1 111 502 1162 727 +965 2 2 1 111 283 1081 505 +966 2 2 1 111 512 865 1229 +967 2 2 1 111 405 1224 644 +968 2 2 1 111 267 1267 553 +969 2 2 1 111 266 554 1268 +970 2 2 1 111 290 1094 769 +971 2 2 1 111 291 770 1095 +972 2 2 1 111 260 542 927 +973 2 2 1 111 261 928 543 +974 2 2 1 111 392 721 747 +975 2 2 1 111 393 748 722 +976 2 2 1 111 432 686 608 +977 2 2 1 111 269 924 1230 +978 2 2 1 111 88 546 1281 +979 2 2 1 111 217 728 704 +980 2 2 1 111 218 705 729 +981 2 2 1 111 227 893 1024 +982 2 2 1 111 305 747 721 +983 2 2 1 111 306 722 748 +984 2 2 1 111 7 163 1245 +985 2 2 1 111 8 1246 168 +986 2 2 1 111 442 850 631 +987 2 2 1 111 427 609 962 +988 2 2 1 111 460 546 1022 +989 2 2 1 111 461 1023 547 +990 2 2 1 111 327 1220 597 +991 2 2 1 111 328 598 1221 +992 2 2 1 111 276 873 590 +993 2 2 1 111 456 674 1176 +994 2 2 1 111 457 1177 673 +995 2 2 1 111 371 631 850 +996 2 2 1 111 88 1022 546 +997 2 2 1 111 116 547 1023 +998 2 2 1 111 490 1139 859 +999 2 2 1 111 270 589 1144 +1000 2 2 1 111 378 704 728 +1001 2 2 1 111 379 729 705 +1002 2 2 1 111 247 645 1105 +1003 2 2 1 111 437 1034 902 +1004 2 2 1 111 438 903 1035 +1005 2 2 1 111 439 790 879 +1006 2 2 1 111 440 880 791 +1007 2 2 1 111 111 618 112 +1008 2 2 1 111 106 1254 515 +1009 2 2 1 111 402 909 800 +1010 2 2 1 111 403 801 910 +1011 2 2 1 111 197 585 1236 +1012 2 2 1 111 198 1237 586 +1013 2 2 1 111 458 1236 585 +1014 2 2 1 111 459 586 1237 +1015 2 2 1 111 362 967 513 +1016 2 2 1 111 229 715 1074 +1017 2 2 1 111 228 1073 714 +1018 2 2 1 111 319 781 800 +1019 2 2 1 111 320 801 782 +1020 2 2 1 111 362 609 711 +1021 2 2 1 111 205 732 1017 +1022 2 2 1 111 265 516 859 +1023 2 2 1 111 435 1125 937 +1024 2 2 1 111 464 1173 671 +1025 2 2 1 111 465 672 1174 +1026 2 2 1 111 206 1085 656 +1027 2 2 1 111 686 865 1206 +1028 2 2 1 111 173 574 723 +1029 2 2 1 111 106 515 889 +1030 2 2 1 111 415 856 613 +1031 2 2 1 111 231 894 548 +1032 2 2 1 111 232 549 895 +1033 2 2 1 111 107 713 108 +1034 2 2 1 111 281 914 1083 +1035 2 2 1 111 282 1084 915 +1036 2 2 1 111 304 756 562 +1037 2 2 1 111 331 548 960 +1038 2 2 1 111 332 959 549 +1039 2 2 1 111 382 796 871 +1040 2 2 1 111 383 872 797 +1041 2 2 1 111 204 1264 563 +1042 2 2 1 111 19 558 1030 +1043 2 2 1 111 53 1031 559 +1044 2 2 1 111 237 1056 848 +1045 2 2 1 111 238 849 1057 +1046 2 2 1 111 147 590 873 +1047 2 2 1 111 146 873 552 +1048 2 2 1 111 145 552 867 +1049 2 2 1 111 277 818 1183 +1050 2 2 1 111 386 652 999 +1051 2 2 1 111 387 1000 653 +1052 2 2 1 111 422 877 575 +1053 2 2 1 111 423 576 878 +1054 2 2 1 111 429 919 1142 +1055 2 2 1 111 348 923 555 +1056 2 2 1 111 324 796 706 +1057 2 2 1 111 325 707 797 +1058 2 2 1 111 319 863 781 +1059 2 2 1 111 320 782 864 +1060 2 2 1 111 483 851 1238 +1061 2 2 1 111 483 712 1111 +1062 2 2 1 111 426 1082 713 +1063 2 2 1 111 70 637 71 +1064 2 2 1 111 133 636 134 +1065 2 2 1 111 101 629 102 +1066 2 2 1 111 321 730 568 +1067 2 2 1 111 322 569 731 +1068 2 2 1 111 19 20 907 +1069 2 2 1 111 52 53 908 +1070 2 2 1 111 307 874 644 +1071 2 2 1 111 333 875 817 +1072 2 2 1 111 268 1209 921 +1073 2 2 1 111 188 1161 726 +1074 2 2 1 111 189 727 1162 +1075 2 2 1 111 313 924 567 +1076 2 2 1 111 265 859 1139 +1077 2 2 1 111 516 625 859 +1078 2 2 1 111 424 774 660 +1079 2 2 1 111 425 661 775 +1080 2 2 1 111 327 907 1220 +1081 2 2 1 111 328 1221 908 +1082 2 2 1 111 348 555 1135 +1083 2 2 1 111 405 616 827 +1084 2 2 1 111 363 556 932 +1085 2 2 1 111 364 933 557 +1086 2 2 1 111 312 566 921 +1087 2 2 1 111 307 739 572 +1088 2 2 1 111 120 651 121 +1089 2 2 1 111 83 650 84 +1090 2 2 1 111 233 671 1173 +1091 2 2 1 111 234 1174 672 +1092 2 2 1 111 618 1190 1088 +1093 2 2 1 111 458 1233 1236 +1094 2 2 1 111 459 1237 1234 +1095 2 2 1 111 601 779 1063 +1096 2 2 1 111 602 1064 780 +1097 2 2 1 111 95 666 1168 +1098 2 2 1 111 214 817 607 +1099 2 2 1 111 203 570 802 +1100 2 2 1 111 433 781 863 +1101 2 2 1 111 434 864 782 +1102 2 2 1 111 295 701 689 +1103 2 2 1 111 296 690 702 +1104 2 2 1 111 480 839 1267 +1105 2 2 1 111 481 1268 840 +1106 2 2 1 111 219 1034 601 +1107 2 2 1 111 220 602 1035 +1108 2 2 1 111 205 1017 1038 +1109 2 2 1 111 269 567 924 +1110 2 2 1 111 513 967 1112 +1111 2 2 1 111 327 558 907 +1112 2 2 1 111 328 908 559 +1113 2 2 1 111 301 608 686 +1114 2 2 1 111 268 921 566 +1115 2 2 1 111 377 789 768 +1116 2 2 1 111 247 866 645 +1117 2 2 1 111 412 664 590 +1118 2 2 1 111 197 1236 568 +1119 2 2 1 111 198 569 1237 +1120 2 2 1 111 251 1014 469 +1121 2 2 1 111 252 470 1015 +1122 2 2 1 111 506 629 1127 +1123 2 2 1 111 289 620 719 +1124 2 2 1 111 288 720 621 +1125 2 2 1 111 445 1168 666 +1126 2 2 1 111 94 666 95 +1127 2 2 1 111 592 1089 827 +1128 2 2 1 111 428 854 816 +1129 2 2 1 111 400 896 1256 +1130 2 2 1 111 221 790 575 +1131 2 2 1 111 222 576 791 +1132 2 2 1 111 363 1002 1163 +1133 2 2 1 111 364 1164 1003 +1134 2 2 1 111 70 776 637 +1135 2 2 1 111 134 636 777 +1136 2 2 1 111 295 610 701 +1137 2 2 1 111 296 702 611 +1138 2 2 1 111 411 904 562 +1139 2 2 1 111 477 1093 712 +1140 2 2 1 111 251 654 1204 +1141 2 2 1 111 252 1205 655 +1142 2 2 1 111 92 608 757 +1143 2 2 1 111 161 860 571 +1144 2 2 1 111 98 913 638 +1145 2 2 1 111 166 573 861 +1146 2 2 1 111 1180 740 925 +1147 2 2 1 111 1181 926 741 +1148 2 2 1 111 179 1089 771 +1149 2 2 1 111 173 723 665 +1150 2 2 1 111 191 1111 712 +1151 2 2 1 111 380 740 1180 +1152 2 2 1 111 381 1181 741 +1153 2 2 1 111 203 997 688 +1154 2 2 1 111 447 626 787 +1155 2 2 1 111 448 788 627 +1156 2 2 1 111 277 913 1096 +1157 2 2 1 111 412 572 739 +1158 2 2 1 111 270 843 589 +1159 2 2 1 111 400 1097 624 +1160 2 2 1 111 201 816 938 +1161 2 2 1 111 411 1088 1190 +1162 2 2 1 111 311 723 574 +1163 2 2 1 111 287 964 619 +1164 2 2 1 111 321 774 846 +1165 2 2 1 111 322 847 775 +1166 2 2 1 111 227 885 893 +1167 2 2 1 111 173 875 574 +1168 2 2 1 111 424 846 774 +1169 2 2 1 111 425 775 847 +1170 2 2 1 111 314 612 1139 +1171 2 2 1 111 312 657 978 +1172 2 2 1 111 506 1105 645 +1173 2 2 1 111 245 1037 1130 +1174 2 2 1 111 446 1050 711 +1175 2 2 1 111 264 594 832 +1176 2 2 1 111 287 997 896 +1177 2 2 1 111 307 644 739 +1178 2 2 1 111 216 593 1046 +1179 2 2 1 111 400 624 1210 +1180 2 2 1 111 281 579 1199 +1181 2 2 1 111 282 1200 580 +1182 2 2 1 111 303 1032 634 +1183 2 2 1 111 304 635 1033 +1184 2 2 1 111 150 648 151 +1185 2 2 1 111 157 649 158 +1186 2 2 1 111 334 1040 603 +1187 2 2 1 111 335 604 1041 +1188 2 2 1 111 415 1105 1127 +1189 2 2 1 111 313 936 617 +1190 2 2 1 111 287 896 646 +1191 2 2 1 111 420 972 954 +1192 2 2 1 111 314 804 612 +1193 2 2 1 111 480 1106 839 +1194 2 2 1 111 481 840 1107 +1195 2 2 1 111 309 577 1028 +1196 2 2 1 111 310 1029 578 +1197 2 2 1 111 317 630 852 +1198 2 2 1 111 207 1112 967 +1199 2 2 1 111 371 1112 631 +1200 2 2 1 111 265 1013 516 +1201 2 2 1 111 301 757 608 +1202 2 2 1 111 121 651 1171 +1203 2 2 1 111 83 1172 650 +1204 2 2 1 111 188 726 599 +1205 2 2 1 111 189 600 727 +1206 2 2 1 111 273 854 564 +1207 2 2 1 111 274 565 855 +1208 2 2 1 111 329 637 776 +1209 2 2 1 111 330 777 636 +1210 2 2 1 111 362 513 1067 +1211 2 2 1 111 136 675 137 +1212 2 2 1 111 67 676 68 +1213 2 2 1 111 334 689 701 +1214 2 2 1 111 335 702 690 +1215 2 2 1 111 525 716 1160 +1216 2 2 1 111 420 758 972 +1217 2 2 1 111 446 1232 1050 +1218 2 2 1 111 300 595 895 +1219 2 2 1 111 299 894 596 +1220 2 2 1 111 276 590 664 +1221 2 2 1 111 280 744 592 +1222 2 2 1 111 171 577 1103 +1223 2 2 1 111 172 1104 578 +1224 2 2 1 111 163 822 1245 +1225 2 2 1 111 168 1246 823 +1226 2 2 1 111 308 802 570 +1227 2 2 1 111 389 1103 577 +1228 2 2 1 111 390 578 1104 +1229 2 2 1 111 556 1187 1213 +1230 2 2 1 111 557 1214 1188 +1231 2 2 1 111 360 897 724 +1232 2 2 1 111 361 725 898 +1233 2 2 1 111 144 867 625 +1234 2 2 1 111 538 1094 848 +1235 2 2 1 111 539 849 1095 +1236 2 2 1 111 155 708 156 +1237 2 2 1 111 380 658 784 +1238 2 2 1 111 381 785 659 +1239 2 2 1 111 283 613 1081 +1240 2 2 1 111 339 564 854 +1241 2 2 1 111 341 855 565 +1242 2 2 1 111 235 784 658 +1243 2 2 1 111 236 659 785 +1244 2 2 1 111 291 632 760 +1245 2 2 1 111 290 759 633 +1246 2 2 1 111 373 848 1094 +1247 2 2 1 111 374 1095 849 +1248 2 2 1 111 549 959 763 +1249 2 2 1 111 548 764 960 +1250 2 2 1 111 342 665 723 +1251 2 2 1 111 235 805 974 +1252 2 2 1 111 236 973 806 +1253 2 2 1 111 473 974 805 +1254 2 2 1 111 474 806 973 +1255 2 2 1 111 46 47 842 +1256 2 2 1 111 25 26 841 +1257 2 2 1 111 309 952 577 +1258 2 2 1 111 310 578 953 +1259 2 2 1 111 174 768 789 +1260 2 2 1 111 318 853 656 +1261 2 2 1 111 114 756 115 +1262 2 2 1 111 416 1209 761 +1263 2 2 1 111 216 1046 563 +1264 2 2 1 111 359 544 1089 +1265 2 2 1 111 424 809 846 +1266 2 2 1 111 425 847 810 +1267 2 2 1 111 186 846 809 +1268 2 2 1 111 187 810 847 +1269 2 2 1 111 334 701 772 +1270 2 2 1 111 335 773 702 +1271 2 2 1 111 409 881 930 +1272 2 2 1 111 410 931 882 +1273 2 2 1 111 122 668 123 +1274 2 2 1 111 81 669 82 +1275 2 2 1 111 294 867 552 +1276 2 2 1 111 483 1111 851 +1277 2 2 1 111 281 1199 605 +1278 2 2 1 111 282 606 1200 +1279 2 2 1 111 312 978 751 +1280 2 2 1 111 130 683 131 +1281 2 2 1 111 73 682 74 +1282 2 2 1 111 377 545 1006 +1283 2 2 1 111 391 645 866 +1284 2 2 1 111 202 591 1006 +1285 2 2 1 111 137 675 833 +1286 2 2 1 111 67 834 676 +1287 2 2 1 111 287 646 964 +1288 2 2 1 111 446 692 695 +1289 2 2 1 111 483 1238 1203 +1290 2 2 1 111 384 677 945 +1291 2 2 1 111 193 1149 566 +1292 2 2 1 111 194 567 1150 +1293 2 2 1 111 333 1061 574 +1294 2 2 1 111 277 638 913 +1295 2 2 1 111 224 871 796 +1296 2 2 1 111 225 797 872 +1297 2 2 1 111 59 60 673 +1298 2 2 1 111 12 13 674 +1299 2 2 1 111 239 718 1004 +1300 2 2 1 111 257 952 1164 +1301 2 2 1 111 258 1163 953 +1302 2 2 1 111 209 960 764 +1303 2 2 1 111 208 763 959 +1304 2 2 1 111 428 816 918 +1305 2 2 1 111 112 618 1088 +1306 2 2 1 111 449 1171 651 +1307 2 2 1 111 450 650 1172 +1308 2 2 1 111 289 941 620 +1309 2 2 1 111 288 621 942 +1310 2 2 1 111 372 798 583 +1311 2 2 1 111 377 591 789 +1312 2 2 1 111 365 620 941 +1313 2 2 1 111 366 942 621 +1314 2 2 1 111 285 662 698 +1315 2 2 1 111 286 699 663 +1316 2 2 1 111 295 1133 614 +1317 2 2 1 111 296 615 1134 +1318 2 2 1 111 368 640 1275 +1319 2 2 1 111 367 1276 639 +1320 2 2 1 111 175 701 610 +1321 2 2 1 111 176 611 702 +1322 2 2 1 111 453 979 1149 +1323 2 2 1 111 454 1150 980 +1324 2 2 1 111 483 1203 712 +1325 2 2 1 111 181 615 950 +1326 2 2 1 111 180 951 614 +1327 2 2 1 111 307 631 874 +1328 2 2 1 111 7 839 1106 +1329 2 2 1 111 8 1107 840 +1330 2 2 1 111 462 1083 914 +1331 2 2 1 111 463 915 1084 +1332 2 2 1 111 92 757 93 +1333 2 2 1 111 318 950 615 +1334 2 2 1 111 317 614 951 +1335 2 2 1 111 175 772 701 +1336 2 2 1 111 176 702 773 +1337 2 2 1 111 242 945 677 +1338 2 2 1 111 202 735 1142 +1339 2 2 1 111 315 994 584 +1340 2 2 1 111 314 876 804 +1341 2 2 1 111 228 740 819 +1342 2 2 1 111 229 820 741 +1343 2 2 1 111 380 819 740 +1344 2 2 1 111 381 741 820 +1345 2 2 1 111 516 1013 619 +1346 2 2 1 111 359 1062 544 +1347 2 2 1 111 420 1070 803 +1348 2 2 1 111 302 692 1047 +1349 2 2 1 111 415 1127 856 +1350 2 2 1 111 246 803 716 +1351 2 2 1 111 319 909 603 +1352 2 2 1 111 320 604 910 +1353 2 2 1 111 405 644 874 +1354 2 2 1 111 362 1050 967 +1355 2 2 1 111 338 571 821 +1356 2 2 1 111 309 1199 579 +1357 2 2 1 111 310 580 1200 +1358 2 2 1 111 6 670 1080 +1359 2 2 1 111 57 58 794 +1360 2 2 1 111 14 15 795 +1361 2 2 1 111 89 700 90 +1362 2 2 1 111 340 824 573 +1363 2 2 1 111 221 575 1277 +1364 2 2 1 111 222 1278 576 +1365 2 2 1 111 544 771 1089 +1366 2 2 1 111 417 762 1230 +1367 2 2 1 111 352 833 675 +1368 2 2 1 111 351 676 834 +1369 2 2 1 111 521 1164 952 +1370 2 2 1 111 522 953 1163 +1371 2 2 1 111 203 612 804 +1372 2 2 1 111 308 718 1231 +1373 2 2 1 111 422 1213 1187 +1374 2 2 1 111 423 1188 1214 +1375 2 2 1 111 162 571 995 +1376 2 2 1 111 167 996 573 +1377 2 2 1 111 152 792 584 +1378 2 2 1 111 422 575 1121 +1379 2 2 1 111 423 1122 576 +1380 2 2 1 111 302 713 889 +1381 2 2 1 111 339 821 571 +1382 2 2 1 111 360 983 897 +1383 2 2 1 111 361 898 984 +1384 2 2 1 111 239 703 818 +1385 2 2 1 111 146 147 873 +1386 2 2 1 111 471 581 970 +1387 2 2 1 111 472 971 582 +1388 2 2 1 111 278 1235 583 +1389 2 2 1 111 341 573 824 +1390 2 2 1 111 336 948 585 +1391 2 2 1 111 337 586 949 +1392 2 2 1 111 144 145 867 +1393 2 2 1 111 273 624 1097 +1394 2 2 1 111 42 43 815 +1395 2 2 1 111 29 30 814 +1396 2 2 1 111 190 1070 954 +1397 2 2 1 111 171 1028 577 +1398 2 2 1 111 172 578 1029 +1399 2 2 1 111 333 574 875 +1400 2 2 1 111 34 35 697 +1401 2 2 1 111 37 38 696 +1402 2 2 1 111 197 1191 585 +1403 2 2 1 111 198 586 1192 +1404 2 2 1 111 339 571 860 +1405 2 2 1 111 427 892 947 +1406 2 2 1 111 235 658 805 +1407 2 2 1 111 236 806 659 +1408 2 2 1 111 475 714 769 +1409 2 2 1 111 476 770 715 +1410 2 2 1 111 336 805 658 +1411 2 2 1 111 337 659 806 +1412 2 2 1 111 341 861 573 +1413 2 2 1 111 281 1136 579 +1414 2 2 1 111 282 580 1137 +1415 2 2 1 111 290 769 714 +1416 2 2 1 111 291 715 770 +1417 2 2 1 111 518 1215 766 +1418 2 2 1 111 519 767 1216 +1419 2 2 1 111 1210 5 964 +1420 2 2 1 111 195 553 1042 +1421 2 2 1 111 196 1043 554 +1422 2 2 1 111 130 836 683 +1423 2 2 1 111 74 682 835 +1424 2 2 1 111 174 789 641 +1425 2 2 1 111 195 603 909 +1426 2 2 1 111 196 910 604 +1427 2 2 1 111 377 1021 545 +1428 2 2 1 111 421 607 817 +1429 2 2 1 111 248 999 581 +1430 2 2 1 111 249 582 1000 +1431 2 2 1 111 279 591 1247 +1432 2 2 1 111 449 651 912 +1433 2 2 1 111 450 911 650 +1434 2 2 1 111 201 918 816 +1435 2 2 1 111 223 609 1067 +1436 2 2 1 111 496 1096 913 +1437 2 2 1 111 175 761 869 +1438 2 2 1 111 176 870 762 +1439 2 2 1 111 110 937 111 +1440 2 2 1 111 141 619 964 +1441 2 2 1 111 362 1067 609 +1442 2 2 1 111 283 765 1280 +1443 2 2 1 111 216 739 644 +1444 2 2 1 111 182 968 914 +1445 2 2 1 111 183 915 969 +1446 2 2 1 111 267 553 1157 +1447 2 2 1 111 266 1156 554 +1448 2 2 1 111 206 811 929 +1449 2 2 1 111 385 1046 593 +1450 2 2 1 111 355 683 836 +1451 2 2 1 111 356 835 682 +1452 2 2 1 111 333 758 901 +1453 2 2 1 111 640 857 1275 +1454 2 2 1 111 639 1276 858 +1455 2 2 1 111 402 1157 553 +1456 2 2 1 111 403 554 1156 +1457 2 2 1 111 347 916 755 +1458 2 2 1 111 464 671 1185 +1459 2 2 1 111 465 1186 672 +1460 2 2 1 111 109 623 1155 +1461 2 2 1 111 308 1231 888 +1462 2 2 1 111 368 636 957 +1463 2 2 1 111 367 958 637 +1464 2 2 1 111 398 1277 877 +1465 2 2 1 111 399 878 1278 +1466 2 2 1 111 575 877 1277 +1467 2 2 1 111 576 1278 878 +1468 2 2 1 111 380 784 819 +1469 2 2 1 111 381 820 785 +1470 2 2 1 111 133 957 636 +1471 2 2 1 111 71 637 958 +1472 2 2 1 111 327 819 784 +1473 2 2 1 111 328 785 820 +1474 2 2 1 111 294 552 1241 +1475 2 2 1 111 323 1279 893 +1476 2 2 1 111 392 1117 587 +1477 2 2 1 111 393 588 1118 +1478 2 2 1 111 193 1196 769 +1479 2 2 1 111 194 770 1197 +1480 2 2 1 111 177 1081 613 +1481 2 2 1 111 491 1279 594 +1482 2 2 1 111 375 716 803 +1483 2 2 1 111 455 1160 716 +1484 2 2 1 111 243 1153 678 +1485 2 2 1 111 244 679 1154 +1486 2 2 1 111 153 584 783 +1487 2 2 1 111 353 1267 839 +1488 2 2 1 111 354 840 1268 +1489 2 2 1 111 21 1114 597 +1490 2 2 1 111 51 598 1113 +1491 2 2 1 111 388 617 1175 +1492 2 2 1 111 286 663 1018 +1493 2 2 1 111 285 1019 662 +1494 2 2 1 111 407 1018 663 +1495 2 2 1 111 408 662 1019 +1496 2 2 1 111 60 61 955 +1497 2 2 1 111 11 12 956 +1498 2 2 1 111 315 792 648 +1499 2 2 1 111 497 622 917 +1500 2 2 1 111 302 695 692 +1501 2 2 1 111 246 901 803 +1502 2 2 1 111 80 965 81 +1503 2 2 1 111 123 966 124 +1504 2 2 1 111 397 917 622 +1505 2 2 1 111 474 1192 806 +1506 2 2 1 111 473 805 1191 +1507 2 2 1 111 345 976 807 +1508 2 2 1 111 346 808 977 +1509 2 2 1 111 382 871 772 +1510 2 2 1 111 383 773 872 +1511 2 2 1 111 358 1024 987 +1512 2 2 1 111 285 746 1019 +1513 2 2 1 111 286 1018 745 +1514 2 2 1 111 501 1161 749 +1515 2 2 1 111 502 750 1162 +1516 2 2 1 111 420 803 901 +1517 2 2 1 111 162 995 163 +1518 2 2 1 111 471 680 1123 +1519 2 2 1 111 472 1124 681 +1520 2 2 1 111 451 1019 746 +1521 2 2 1 111 452 745 1018 +1522 2 2 1 111 167 168 996 +1523 2 2 1 111 316 649 793 +1524 2 2 1 111 397 622 922 +1525 2 2 1 111 182 930 881 +1526 2 2 1 111 183 882 931 +1527 2 2 1 111 462 914 968 +1528 2 2 1 111 463 969 915 +1529 2 2 1 111 69 776 70 +1530 2 2 1 111 134 777 135 +1531 2 2 1 111 303 728 1255 +1532 2 2 1 111 45 46 1065 +1533 2 2 1 111 26 27 1066 +1534 2 2 1 111 174 641 830 +1535 2 2 1 111 294 625 867 +1536 2 2 1 111 237 1077 1056 +1537 2 2 1 111 238 1057 1078 +1538 2 2 1 111 221 905 1108 +1539 2 2 1 111 222 1109 906 +1540 2 2 1 111 323 893 628 +1541 2 2 1 111 214 758 817 +1542 2 2 1 111 385 563 1046 +1543 2 2 1 111 53 54 1031 +1544 2 2 1 111 18 19 1030 +1545 2 2 1 111 334 871 1040 +1546 2 2 1 111 335 1041 872 +1547 2 2 1 111 349 981 828 +1548 2 2 1 111 350 829 982 +1549 2 2 1 111 191 1093 628 +1550 2 2 1 111 728 1217 1255 +1551 2 2 1 111 324 706 1248 +1552 2 2 1 111 325 1249 707 +1553 2 2 1 111 326 641 1152 +1554 2 2 1 111 445 647 1257 +1555 2 2 1 111 375 1079 716 +1556 2 2 1 111 122 1171 668 +1557 2 2 1 111 82 669 1172 +1558 2 2 1 111 88 1281 89 +1559 2 2 1 111 207 967 616 +1560 2 2 1 111 326 830 641 +1561 2 2 1 111 347 884 916 +1562 2 2 1 111 432 1217 686 +1563 2 2 1 111 491 987 1024 +1564 2 2 1 111 103 1145 104 +1565 2 2 1 111 313 752 936 +1566 2 2 1 111 5 1210 624 +1567 2 2 1 111 237 706 1077 +1568 2 2 1 111 238 1078 707 +1569 2 2 1 111 555 923 642 +1570 2 2 1 111 359 1089 592 +1571 2 2 1 111 207 631 1112 +1572 2 2 1 111 365 651 934 +1573 2 2 1 111 366 935 650 +1574 2 2 1 111 437 902 753 +1575 2 2 1 111 438 754 903 +1576 2 2 1 111 324 1066 845 +1577 2 2 1 111 325 844 1065 +1578 2 2 1 111 510 1123 680 +1579 2 2 1 111 511 681 1124 +1580 2 2 1 111 120 934 651 +1581 2 2 1 111 84 650 935 +1582 2 2 1 111 173 665 798 +1583 2 2 1 111 56 1131 663 +1584 2 2 1 111 16 662 1132 +1585 2 2 1 111 418 601 1273 +1586 2 2 1 111 419 1274 602 +1587 2 2 1 111 345 753 902 +1588 2 2 1 111 346 903 754 +1589 2 2 1 111 85 935 621 +1590 2 2 1 111 119 620 934 +1591 2 2 1 111 441 929 811 +1592 2 2 1 111 413 869 761 +1593 2 2 1 111 414 762 870 +1594 2 2 1 111 59 673 1177 +1595 2 2 1 111 13 1176 674 +1596 2 2 1 111 284 1127 629 +1597 2 2 1 111 482 802 691 +1598 2 2 1 111 56 57 1131 +1599 2 2 1 111 15 16 1132 +1600 2 2 1 111 111 937 618 +1601 2 2 1 111 370 947 892 +1602 2 2 1 111 308 691 802 +1603 2 2 1 111 411 1190 642 +1604 2 2 1 111 466 1073 597 +1605 2 2 1 111 467 598 1074 +1606 2 2 1 111 365 934 620 +1607 2 2 1 111 366 621 935 +1608 2 2 1 111 66 834 67 +1609 2 2 1 111 137 833 138 +1610 2 2 1 111 527 992 660 +1611 2 2 1 111 528 661 993 +1612 2 2 1 111 177 856 1027 +1613 2 2 1 111 468 1209 868 +1614 2 2 1 111 488 941 693 +1615 2 2 1 111 489 694 942 +1616 2 2 1 111 170 594 1175 +1617 2 2 1 111 231 660 992 +1618 2 2 1 111 232 993 661 +1619 2 2 1 111 369 1004 570 +1620 2 2 1 111 21 597 1220 +1621 2 2 1 111 51 1221 598 +1622 2 2 1 111 475 766 1115 +1623 2 2 1 111 476 1116 767 +1624 2 2 1 111 129 836 130 +1625 2 2 1 111 74 835 75 +1626 2 2 1 111 466 597 1114 +1627 2 2 1 111 467 1113 598 +1628 2 2 1 111 443 1258 599 +1629 2 2 1 111 444 600 1259 +1630 2 2 1 111 343 1115 766 +1631 2 2 1 111 344 767 1116 +1632 2 2 1 111 455 716 1079 +1633 2 2 1 111 209 639 858 +1634 2 2 1 111 208 857 640 +1635 2 2 1 111 394 1063 779 +1636 2 2 1 111 395 780 1064 +1637 2 2 1 111 287 619 1013 +1638 2 2 1 111 377 1006 591 +1639 2 2 1 111 219 601 1063 +1640 2 2 1 111 220 1064 602 +1641 2 2 1 111 398 599 1194 +1642 2 2 1 111 399 1195 600 +1643 2 2 1 111 424 886 809 +1644 2 2 1 111 425 810 887 +1645 2 2 1 111 90 700 1110 +1646 2 2 1 111 420 901 758 +1647 2 2 1 111 302 1047 713 +1648 2 2 1 111 700 1255 1110 +1649 2 2 1 111 380 1180 658 +1650 2 2 1 111 381 659 1181 +1651 2 2 1 111 226 884 837 +1652 2 2 1 111 227 838 885 +1653 2 2 1 111 109 1155 110 +1654 2 2 1 111 376 818 703 +1655 2 2 1 111 406 1130 1037 +1656 2 2 1 111 357 963 1016 +1657 2 2 1 111 294 859 625 +1658 2 2 1 111 464 1185 1136 +1659 2 2 1 111 465 1137 1186 +1660 2 2 1 111 401 1184 657 +1661 2 2 1 111 409 837 884 +1662 2 2 1 111 410 885 838 +1663 2 2 1 111 409 704 881 +1664 2 2 1 111 410 882 705 +1665 2 2 1 111 267 899 1219 +1666 2 2 1 111 266 1218 900 +1667 2 2 1 111 352 675 1053 +1668 2 2 1 111 351 1054 676 +1669 2 2 1 111 197 746 1191 +1670 2 2 1 111 198 1192 745 +1671 2 2 1 111 226 837 724 +1672 2 2 1 111 227 725 838 +1673 2 2 1 111 338 995 571 +1674 2 2 1 111 517 922 622 +1675 2 2 1 111 160 860 161 +1676 2 2 1 111 165 166 861 +1677 2 2 1 111 275 1190 618 +1678 2 2 1 111 340 573 996 +1679 2 2 1 111 376 961 638 +1680 2 2 1 111 170 1175 617 +1681 2 2 1 111 44 45 844 +1682 2 2 1 111 27 28 845 +1683 2 2 1 111 218 729 904 +1684 2 2 1 111 110 1155 937 +1685 2 2 1 111 153 783 154 +1686 2 2 1 111 394 890 812 +1687 2 2 1 111 395 813 891 +1688 2 2 1 111 378 881 704 +1689 2 2 1 111 379 705 882 +1690 2 2 1 111 475 769 1196 +1691 2 2 1 111 476 1197 770 +1692 2 2 1 111 360 724 837 +1693 2 2 1 111 361 838 725 +1694 2 2 1 111 219 1262 902 +1695 2 2 1 111 220 903 1263 +1696 2 2 1 111 384 883 1239 +1697 2 2 1 111 347 865 1098 +1698 2 2 1 111 398 825 905 +1699 2 2 1 111 399 906 826 +1700 2 2 1 111 439 879 737 +1701 2 2 1 111 440 738 880 +1702 2 2 1 111 109 1082 623 +1703 2 2 1 111 349 737 879 +1704 2 2 1 111 350 880 738 +1705 2 2 1 111 294 1241 643 +1706 2 2 1 111 426 623 1082 +1707 2 2 1 111 333 817 758 +1708 2 2 1 111 163 995 822 +1709 2 2 1 111 105 1254 106 +1710 2 2 1 111 495 1198 1126 +1711 2 2 1 111 168 823 996 +1712 2 2 1 111 207 874 631 +1713 2 2 1 111 417 811 853 +1714 2 2 1 111 206 853 811 +1715 2 2 1 111 219 1058 1262 +1716 2 2 1 111 220 1263 1059 +1717 2 2 1 111 317 1252 610 +1718 2 2 1 111 318 611 1253 +1719 2 2 1 111 255 652 1048 +1720 2 2 1 111 256 1049 653 +1721 2 2 1 111 409 930 837 +1722 2 2 1 111 410 838 931 +1723 2 2 1 111 386 1048 652 +1724 2 2 1 111 387 653 1049 +1725 2 2 1 111 116 1023 117 +1726 2 2 1 111 87 1022 88 +1727 2 2 1 111 426 713 1047 +1728 2 2 1 111 170 617 936 +1729 2 2 1 111 473 1191 746 +1730 2 2 1 111 474 745 1192 +1731 2 2 1 111 323 628 1093 +1732 2 2 1 111 106 889 107 +1733 2 2 1 111 204 643 1264 +1734 2 2 1 111 228 819 1073 +1735 2 2 1 111 229 1074 820 +1736 2 2 1 111 274 735 850 +1737 2 2 1 111 151 792 152 +1738 2 2 1 111 287 1013 688 +1739 2 2 1 111 455 1016 963 +1740 2 2 1 111 374 1025 632 +1741 2 2 1 111 373 633 1026 +1742 2 2 1 111 206 929 717 +1743 2 2 1 111 366 650 911 +1744 2 2 1 111 365 912 651 +1745 2 2 1 111 422 1187 877 +1746 2 2 1 111 423 878 1188 +1747 2 2 1 111 267 1219 480 +1748 2 2 1 111 266 481 1218 +1749 2 2 1 111 218 642 923 +1750 2 2 1 111 646 1210 964 +1751 2 2 1 111 469 1014 839 +1752 2 2 1 111 470 840 1015 +1753 2 2 1 111 355 985 683 +1754 2 2 1 111 356 682 986 +1755 2 2 1 111 156 793 157 +1756 2 2 1 111 349 879 1108 +1757 2 2 1 111 350 1109 880 +1758 2 2 1 111 348 1135 628 +1759 2 2 1 111 558 1101 1030 +1760 2 2 1 111 559 1031 1102 +1761 2 2 1 111 435 1155 623 +1762 2 2 1 111 175 869 772 +1763 2 2 1 111 176 773 870 +1764 2 2 1 111 430 1030 1101 +1765 2 2 1 111 431 1102 1031 +1766 2 2 1 111 212 1071 787 +1767 2 2 1 111 213 788 1072 +1768 2 2 1 111 507 1027 856 +1769 2 2 1 111 474 973 1212 +1770 2 2 1 111 473 1211 974 +1771 2 2 1 111 360 930 680 +1772 2 2 1 111 361 681 931 +1773 2 2 1 111 494 936 752 +1774 2 2 1 111 557 933 1214 +1775 2 2 1 111 556 1213 932 +1776 2 2 1 111 370 892 1238 +1777 2 2 1 111 316 793 708 +1778 2 2 1 111 230 736 831 +1779 2 2 1 111 173 998 875 +1780 2 2 1 111 266 900 696 +1781 2 2 1 111 267 697 899 +1782 2 2 1 111 373 1094 633 +1783 2 2 1 111 374 632 1095 +1784 2 2 1 111 182 680 930 +1785 2 2 1 111 183 931 681 +1786 2 2 1 111 495 717 944 +1787 2 2 1 111 279 1152 641 +1788 2 2 1 111 400 646 896 +1789 2 2 1 111 315 1219 899 +1790 2 2 1 111 316 900 1218 +1791 2 2 1 111 24 1026 633 +1792 2 2 1 111 48 632 1025 +1793 2 2 1 111 359 831 866 +1794 2 2 1 111 523 777 1266 +1795 2 2 1 111 524 1265 776 +1796 2 2 1 111 263 1045 1259 +1797 2 2 1 111 262 1258 1044 +1798 2 2 1 111 169 657 1184 +1799 2 2 1 111 104 946 105 +1800 2 2 1 111 342 1138 665 +1801 2 2 1 111 497 917 1090 +1802 2 2 1 111 311 862 1193 +1803 2 2 1 111 357 897 1091 +1804 2 2 1 111 358 1092 898 +1805 2 2 1 111 84 935 85 +1806 2 2 1 111 119 934 120 +1807 2 2 1 111 369 804 876 +1808 2 2 1 111 179 593 1224 +1809 2 2 1 111 21 22 1114 +1810 2 2 1 111 50 51 1113 +1811 2 2 1 111 336 658 939 +1812 2 2 1 111 337 940 659 +1813 2 2 1 111 442 1080 670 +1814 2 2 1 111 290 633 1094 +1815 2 2 1 111 291 1095 632 +1816 2 2 1 111 177 1226 1081 +1817 2 2 1 111 178 1090 917 +1818 2 2 1 111 297 1273 1034 +1819 2 2 1 111 298 1035 1274 +1820 2 2 1 111 180 1147 626 +1821 2 2 1 111 181 627 1148 +1822 2 2 1 111 205 1038 868 +1823 2 2 1 111 601 1034 1273 +1824 2 2 1 111 602 1274 1035 +1825 2 2 1 111 495 1126 830 +1826 2 2 1 111 191 712 1093 +1827 2 2 1 111 377 768 1189 +1828 2 2 1 111 333 901 1061 +1829 2 2 1 111 418 1009 779 +1830 2 2 1 111 419 780 1010 +1831 2 2 1 111 233 634 1032 +1832 2 2 1 111 234 1033 635 +1833 2 2 1 111 390 1251 1086 +1834 2 2 1 111 389 1087 1250 +1835 2 2 1 111 186 779 1009 +1836 2 2 1 111 187 1010 780 +1837 2 2 1 111 191 628 1135 +1838 2 2 1 111 376 667 961 +1839 2 2 1 111 406 962 1189 +1840 2 2 1 111 595 1086 1251 +1841 2 2 1 111 596 1250 1087 +1842 2 2 1 111 376 638 1183 +1843 2 2 1 111 132 957 133 +1844 2 2 1 111 71 958 72 +1845 2 2 1 111 334 772 871 +1846 2 2 1 111 335 872 773 +1847 2 2 1 111 175 610 1119 +1848 2 2 1 111 176 1120 611 +1849 2 2 1 111 527 1103 992 +1850 2 2 1 111 528 993 1104 +1851 2 2 1 111 245 1130 943 +1852 2 2 1 111 217 686 1217 +1853 2 2 1 111 406 1189 768 +1854 2 2 1 111 390 1104 993 +1855 2 2 1 111 389 992 1103 +1856 2 2 1 111 277 1183 638 +1857 2 2 1 111 339 854 1001 +1858 2 2 1 111 341 1005 855 +1859 2 2 1 111 388 717 929 +1860 2 2 1 111 482 896 997 +1861 2 2 1 111 99 961 100 +1862 2 2 1 111 557 1162 750 +1863 2 2 1 111 556 749 1161 +1864 2 2 1 111 107 889 713 +1865 2 2 1 111 210 970 721 +1866 2 2 1 111 211 722 971 +1867 2 2 1 111 357 724 897 +1868 2 2 1 111 358 898 725 +1869 2 2 1 111 240 1260 932 +1870 2 2 1 111 241 933 1261 +1871 2 2 1 111 275 642 1190 +1872 2 2 1 111 35 36 1020 +1873 2 2 1 111 224 1007 863 +1874 2 2 1 111 225 864 1008 +1875 2 2 1 111 382 1077 706 +1876 2 2 1 111 383 707 1078 +1877 2 2 1 111 367 639 1045 +1878 2 2 1 111 368 1044 640 +1879 2 2 1 111 150 1106 648 +1880 2 2 1 111 158 649 1107 +1881 2 2 1 111 210 983 970 +1882 2 2 1 111 211 971 984 +1883 2 2 1 111 60 955 673 +1884 2 2 1 111 12 674 956 +1885 2 2 1 111 330 636 1275 +1886 2 2 1 111 329 1276 637 +1887 2 2 1 111 240 1208 1260 +1888 2 2 1 111 241 1261 1207 +1889 2 2 1 111 421 875 998 +1890 2 2 1 111 108 1082 109 +1891 2 2 1 111 382 772 869 +1892 2 2 1 111 383 870 773 +1893 2 2 1 111 471 970 983 +1894 2 2 1 111 472 984 971 +1895 2 2 1 111 99 638 961 +1896 2 2 1 111 391 831 736 +1897 2 2 1 111 351 1058 812 +1898 2 2 1 111 352 813 1059 +1899 2 2 1 111 352 1053 891 +1900 2 2 1 111 351 890 1054 +1901 2 2 1 111 184 925 1068 +1902 2 2 1 111 185 1069 926 +1903 2 2 1 111 347 755 1229 +1904 2 2 1 111 81 965 669 +1905 2 2 1 111 123 668 966 +1906 2 2 1 111 151 648 792 +1907 2 2 1 111 233 1271 634 +1908 2 2 1 111 234 635 1272 +1909 2 2 1 111 36 37 1182 +1910 2 2 1 111 57 794 1131 +1911 2 2 1 111 15 1132 795 +1912 2 2 1 111 468 868 1038 +1913 2 2 1 111 338 1011 654 +1914 2 2 1 111 340 655 1012 +1915 2 2 1 111 5 141 964 +1916 2 2 1 111 493 1269 751 +1917 2 2 1 111 494 752 1270 +1918 2 2 1 111 560 1133 1178 +1919 2 2 1 111 561 1179 1134 +1920 2 2 1 111 24 25 1026 +1921 2 2 1 111 47 48 1025 +1922 2 2 1 111 289 693 941 +1923 2 2 1 111 288 942 694 +1924 2 2 1 111 518 766 1196 +1925 2 2 1 111 519 1197 767 +1926 2 2 1 111 285 698 1211 +1927 2 2 1 111 286 1212 699 +1928 2 2 1 111 253 751 1269 +1929 2 2 1 111 254 1270 752 +1930 2 2 1 111 343 1215 684 +1931 2 2 1 111 344 685 1216 +1932 2 2 1 111 278 665 1138 +1933 2 2 1 111 311 1193 1017 +1934 2 2 1 111 157 793 649 +1935 2 2 1 111 408 991 795 +1936 2 2 1 111 407 794 990 +1937 2 2 1 111 456 795 991 +1938 2 2 1 111 457 990 794 +1939 2 2 1 111 365 1208 912 +1940 2 2 1 111 366 911 1207 +1941 2 2 1 111 228 1115 740 +1942 2 2 1 111 229 741 1116 +1943 2 2 1 111 267 480 1267 +1944 2 2 1 111 266 1268 481 +1945 2 2 1 111 287 688 997 +1946 2 2 1 111 345 1227 753 +1947 2 2 1 111 346 754 1228 +1948 2 2 1 111 375 755 916 +1949 2 2 1 111 357 1016 724 +1950 2 2 1 111 362 711 1050 +1951 2 2 1 111 42 815 1170 +1952 2 2 1 111 30 1169 814 +1953 2 2 1 111 355 825 985 +1954 2 2 1 111 356 986 826 +1955 2 2 1 111 358 725 1024 +1956 2 2 1 111 169 978 657 +1957 2 2 1 111 206 717 1085 +1958 2 2 1 111 368 1275 636 +1959 2 2 1 111 367 637 1276 +1960 2 2 1 111 658 1180 939 +1961 2 2 1 111 659 940 1181 +1962 2 2 1 111 532 986 682 +1963 2 2 1 111 531 683 985 +1964 2 2 1 111 396 945 920 +1965 2 2 1 111 348 1099 923 +1966 2 2 1 111 343 925 740 +1967 2 2 1 111 344 741 926 +1968 2 2 1 111 447 787 1071 +1969 2 2 1 111 448 1072 788 +1970 2 2 1 111 433 863 1007 +1971 2 2 1 111 434 1008 864 +1972 2 2 1 111 251 709 1014 +1973 2 2 1 111 252 1015 710 +1974 2 2 1 111 314 778 1226 +1975 2 2 1 111 448 1247 919 +1976 2 2 1 111 331 809 886 +1977 2 2 1 111 332 887 810 +1978 2 2 1 111 626 1147 787 +1979 2 2 1 111 627 788 1148 +1980 2 2 1 111 422 1121 733 +1981 2 2 1 111 423 734 1122 +1982 2 2 1 111 201 938 799 +1983 2 2 1 111 292 733 1121 +1984 2 2 1 111 293 1122 734 +1985 2 2 1 111 265 688 1013 +1986 2 2 1 111 315 1020 994 +1987 2 2 1 111 447 918 1235 +1988 2 2 1 111 349 1201 737 +1989 2 2 1 111 350 738 1202 +1990 2 2 1 111 504 1166 1234 +1991 2 2 1 111 503 1233 1165 +1992 2 2 1 111 388 944 717 +1993 2 2 1 111 537 1090 1231 +1994 2 2 1 111 372 998 798 +1995 2 2 1 111 351 812 890 +1996 2 2 1 111 352 891 813 +1997 2 2 1 111 594 1279 832 +1998 2 2 1 111 148 149 1036 +1999 2 2 1 111 430 1211 698 +2000 2 2 1 111 431 699 1212 +2001 2 2 1 111 58 1177 794 +2002 2 2 1 111 14 795 1176 +2003 2 2 1 111 495 944 1198 +2004 2 2 1 111 266 696 1156 +2005 2 2 1 111 267 1157 697 +2006 2 2 1 111 418 678 1009 +2007 2 2 1 111 419 1010 679 +2008 2 2 1 111 391 736 946 +2009 2 2 1 111 457 794 1177 +2010 2 2 1 111 456 1176 795 +2011 2 2 1 111 156 708 793 +2012 2 2 1 111 345 807 1227 +2013 2 2 1 111 346 1228 808 +2014 2 2 1 111 100 961 667 +2015 2 2 1 111 396 920 799 +2016 2 2 1 111 500 1100 1047 +2017 2 2 1 111 404 954 972 +2018 2 2 1 111 338 821 1011 +2019 2 2 1 111 125 1201 126 +2020 2 2 1 111 78 1202 79 +2021 2 2 1 111 355 905 825 +2022 2 2 1 111 356 826 906 +2023 2 2 1 111 340 1012 824 +2024 2 2 1 111 575 790 1121 +2025 2 2 1 111 576 1122 791 +2026 2 2 1 111 428 1001 854 +2027 2 2 1 111 429 855 1005 +2028 2 2 1 111 292 1121 790 +2029 2 2 1 111 293 791 1122 +2030 2 2 1 111 237 848 1075 +2031 2 2 1 111 238 1076 849 +2032 2 2 1 111 371 850 735 +2033 2 2 1 111 316 1218 649 +2034 2 2 1 111 315 648 1219 +2035 2 2 1 111 17 18 1222 +2036 2 2 1 111 54 55 1223 +2037 2 2 1 111 112 1088 113 +2038 2 2 1 111 9 10 1227 +2039 2 2 1 111 62 63 1228 +2040 2 2 1 111 587 1269 1091 +2041 2 2 1 111 588 1092 1270 +2042 2 2 1 111 377 1189 1021 +2043 2 2 1 111 493 1091 1269 +2044 2 2 1 111 494 1270 1092 +2045 2 2 1 111 523 1053 675 +2046 2 2 1 111 524 676 1054 +2047 2 2 1 111 7 1245 839 +2048 2 2 1 111 8 840 1246 +2049 2 2 1 111 478 1178 1133 +2050 2 2 1 111 479 1134 1179 +2051 2 2 1 111 416 852 868 +2052 2 2 1 111 505 1226 778 +2053 2 2 1 111 205 868 852 +2054 2 2 1 111 270 1144 647 +2055 2 2 1 111 372 799 920 +2056 2 2 1 111 400 1210 646 +2057 2 2 1 111 359 866 1062 +2058 2 2 1 111 177 876 1226 +2059 2 2 1 111 253 1117 684 +2060 2 2 1 111 254 685 1118 +2061 2 2 1 111 90 1110 91 +2062 2 2 1 111 392 684 1117 +2063 2 2 1 111 393 1118 685 +2064 2 2 1 111 315 584 792 +2065 2 2 1 111 274 1142 735 +2066 2 2 1 111 415 613 1225 +2067 2 2 1 111 343 766 1215 +2068 2 2 1 111 344 1216 767 +2069 2 2 1 111 445 687 1168 +2070 2 2 1 111 251 1178 709 +2071 2 2 1 111 252 710 1179 +2072 2 2 1 111 407 663 1131 +2073 2 2 1 111 408 1132 662 +2074 2 2 1 111 312 921 1244 +2075 2 2 1 111 258 749 1163 +2076 2 2 1 111 257 1164 750 +2077 2 2 1 111 413 1056 1077 +2078 2 2 1 111 414 1078 1057 +2079 2 2 1 111 214 972 758 +2080 2 2 1 111 312 1244 657 +2081 2 2 1 111 246 862 1061 +2082 2 2 1 111 392 1068 684 +2083 2 2 1 111 393 685 1069 +2084 2 2 1 111 95 1168 96 +2085 2 2 1 111 30 31 1169 +2086 2 2 1 111 41 42 1170 +2087 2 2 1 111 391 866 831 +2088 2 2 1 111 349 828 1201 +2089 2 2 1 111 350 1202 829 +2090 2 2 1 111 394 1158 890 +2091 2 2 1 111 395 891 1159 +2092 2 2 1 111 360 837 930 +2093 2 2 1 111 361 931 838 +2094 2 2 1 111 274 855 1142 +2095 2 2 1 111 7 1106 150 +2096 2 2 1 111 8 158 1107 +2097 2 2 1 111 493 751 978 +2098 2 2 1 111 435 937 1155 +2099 2 2 1 111 226 724 1016 +2100 2 2 1 111 51 52 1221 +2101 2 2 1 111 20 21 1220 +2102 2 2 1 111 121 1171 122 +2103 2 2 1 111 82 1172 83 +2104 2 2 1 111 13 14 1176 +2105 2 2 1 111 58 59 1177 +2106 2 2 1 111 182 881 968 +2107 2 2 1 111 183 969 882 +2108 2 2 1 111 326 656 1085 +2109 2 2 1 111 227 1024 725 +2110 2 2 1 111 382 869 1077 +2111 2 2 1 111 383 1078 870 +2112 2 2 1 111 263 826 986 +2113 2 2 1 111 262 985 825 +2114 2 2 1 111 432 1110 1255 +2115 2 2 1 111 264 1198 944 +2116 2 2 1 111 495 1240 717 +2117 2 2 1 111 17 1222 698 +2118 2 2 1 111 55 699 1223 +2119 2 2 1 111 426 1047 1100 +2120 2 2 1 111 409 1098 704 +2121 2 2 1 111 410 705 1099 +2122 2 2 1 111 398 905 1277 +2123 2 2 1 111 399 1278 906 +2124 2 2 1 111 155 1143 708 +2125 2 2 1 111 245 892 1037 +2126 2 2 1 111 202 1006 1167 +2127 2 2 1 111 226 1016 1079 +2128 2 2 1 111 433 1007 814 +2129 2 2 1 111 434 815 1008 +2130 2 2 1 111 154 1143 155 +2131 2 2 1 111 238 1249 1076 +2132 2 2 1 111 237 1075 1248 +2133 2 2 1 111 173 798 998 +2134 2 2 1 111 392 747 1068 +2135 2 2 1 111 393 1069 748 +2136 2 2 1 111 184 1068 747 +2137 2 2 1 111 185 748 1069 +2138 2 2 1 111 174 975 768 +2139 2 2 1 111 226 916 884 +2140 2 2 1 111 430 698 1222 +2141 2 2 1 111 431 1223 699 +2142 2 2 1 111 276 1264 1241 +2143 2 2 1 111 498 814 1007 +2144 2 2 1 111 499 1008 815 +2145 2 2 1 111 96 1168 687 +2146 2 2 1 111 175 1119 761 +2147 2 2 1 111 176 762 1120 +2148 2 2 1 111 374 842 1025 +2149 2 2 1 111 373 1026 841 +2150 2 2 1 111 215 1243 1256 +2151 2 2 1 111 445 1257 687 +2152 2 2 1 111 396 883 945 +2153 2 2 1 111 477 712 1203 +2154 2 2 1 111 375 1070 755 +2155 2 2 1 111 311 1017 732 +2156 2 2 1 111 514 1256 1243 +2157 2 2 1 111 178 1231 1090 +2158 2 2 1 111 462 968 1271 +2159 2 2 1 111 463 1272 969 +2160 2 2 1 111 416 761 1119 +2161 2 2 1 111 417 1120 762 +2162 2 2 1 111 544 1062 786 +2163 2 2 1 111 436 1126 1198 +2164 2 2 1 111 301 686 1206 +2165 2 2 1 111 391 1145 645 +2166 2 2 1 111 537 1231 718 +2167 2 2 1 111 231 1250 894 +2168 2 2 1 111 232 895 1251 +2169 2 2 1 111 396 799 938 +2170 2 2 1 111 242 920 945 +2171 2 2 1 111 508 1075 841 +2172 2 2 1 111 509 842 1076 +2173 2 2 1 111 326 1240 830 +2174 2 2 1 111 339 1001 821 +2175 2 2 1 111 341 824 1005 +2176 2 2 1 111 253 684 1215 +2177 2 2 1 111 254 1216 685 +2178 2 2 1 111 407 1131 794 +2179 2 2 1 111 408 795 1132 +2180 2 2 1 111 353 1014 709 +2181 2 2 1 111 354 710 1015 +2182 2 2 1 111 505 1081 1226 +2183 2 2 1 111 428 918 1071 +2184 2 2 1 111 429 1072 919 +2185 2 2 1 111 186 1009 846 +2186 2 2 1 111 187 847 1010 +2187 2 2 1 111 373 841 1075 +2188 2 2 1 111 374 1076 842 +2189 2 2 1 111 199 1185 694 +2190 2 2 1 111 200 693 1186 +2191 2 2 1 111 247 1225 786 +2192 2 2 1 111 6 1242 149 +2193 2 2 1 111 36 1182 526 +2194 2 2 1 111 149 1242 1036 +2195 2 2 1 111 190 755 1070 +2196 2 2 1 111 434 1170 815 +2197 2 2 1 111 433 814 1169 +2198 2 2 1 111 401 1038 1017 +2199 2 2 1 111 378 968 881 +2200 2 2 1 111 379 882 969 +2201 2 2 1 111 512 1206 865 +2202 2 2 1 111 338 822 995 +2203 2 2 1 111 340 996 823 +2204 2 2 1 111 394 1140 1158 +2205 2 2 1 111 395 1159 1141 +2206 2 2 1 111 478 1042 709 +2207 2 2 1 111 479 710 1043 +2208 2 2 1 111 210 897 983 +2209 2 2 1 111 211 984 898 +2210 2 2 1 111 504 1234 989 +2211 2 2 1 111 503 988 1233 +2212 2 2 1 111 283 1280 771 +2213 2 2 1 111 428 1071 1001 +2214 2 2 1 111 429 1005 1072 +2215 2 2 1 111 37 696 900 +2216 2 2 1 111 35 899 697 +2217 2 2 1 111 643 1241 1264 +2218 2 2 1 111 422 733 1213 +2219 2 2 1 111 423 1214 734 +2220 2 2 1 111 240 1213 733 +2221 2 2 1 111 241 734 1214 +2222 2 2 1 111 371 735 1167 +2223 2 2 1 111 303 1255 700 +2224 2 2 1 111 343 740 1115 +2225 2 2 1 111 344 1116 741 +2226 2 2 1 111 212 1011 821 +2227 2 2 1 111 213 824 1012 +2228 2 2 1 111 6 1080 1242 +2229 2 2 1 111 406 768 975 +2230 2 2 1 111 514 1243 938 +2231 2 2 1 111 375 803 1070 +2232 2 2 1 111 231 992 1250 +2233 2 2 1 111 232 1251 993 +2234 2 2 1 111 260 1054 890 +2235 2 2 1 111 261 891 1053 +2236 2 2 1 111 451 730 1153 +2237 2 2 1 111 452 1154 731 +2238 2 2 1 111 345 1262 976 +2239 2 2 1 111 346 977 1263 +2240 2 2 1 111 226 1079 916 +2241 2 2 1 111 245 1203 1238 +2242 2 2 1 111 219 812 1058 +2243 2 2 1 111 220 1059 813 +2244 2 2 1 111 529 976 1262 +2245 2 2 1 111 530 1263 977 +2246 2 2 1 111 242 1039 920 +2247 2 2 1 111 550 1158 1140 +2248 2 2 1 111 551 1141 1159 +2249 2 2 1 111 219 1063 812 +2250 2 2 1 111 220 813 1064 +2251 2 2 1 111 89 1281 700 +2252 2 2 1 111 394 812 1063 +2253 2 2 1 111 395 1064 813 +2254 2 2 1 111 413 979 1056 +2255 2 2 1 111 414 1057 980 +2256 2 2 1 111 353 839 1014 +2257 2 2 1 111 354 1015 840 +2258 2 2 1 111 126 1201 828 +2259 2 2 1 111 78 829 1202 +2260 2 2 1 111 47 1025 842 +2261 2 2 1 111 25 841 1026 +2262 2 2 1 111 9 1227 807 +2263 2 2 1 111 63 808 1228 +2264 2 2 1 111 515 1254 736 +2265 2 2 1 111 394 779 1140 +2266 2 2 1 111 395 1141 780 +2267 2 2 1 111 384 945 883 +2268 2 2 1 111 217 1217 728 +2269 2 2 1 111 455 1079 1016 +2270 2 2 1 111 212 1001 1071 +2271 2 2 1 111 213 1072 1005 +2272 2 2 1 111 370 851 1060 +2273 2 2 1 111 385 771 1280 +2274 2 2 1 111 363 1163 749 +2275 2 2 1 111 364 750 1164 +2276 2 2 1 111 125 737 1201 +2277 2 2 1 111 79 1202 738 +2278 2 2 1 111 237 1248 706 +2279 2 2 1 111 238 707 1249 +2280 2 2 1 111 224 863 1040 +2281 2 2 1 111 225 1041 864 +2282 2 2 1 111 429 1142 855 +2283 2 2 1 111 215 883 1243 +2284 2 2 1 111 280 1050 1232 +2285 2 2 1 111 188 877 1187 +2286 2 2 1 111 189 1188 878 +2287 2 2 1 111 545 1167 1006 +2288 2 2 1 111 280 1232 744 +2289 2 2 1 111 202 1167 735 +2290 2 2 1 111 386 1051 1048 +2291 2 2 1 111 387 1049 1052 +2292 2 2 1 111 45 1065 844 +2293 2 2 1 111 27 845 1066 +2294 2 2 1 111 403 1156 696 +2295 2 2 1 111 402 697 1157 +2296 2 2 1 111 503 1048 1051 +2297 2 2 1 111 504 1052 1049 +2298 2 2 1 111 186 1140 779 +2299 2 2 1 111 187 780 1141 +2300 2 2 1 111 191 1135 1151 +2301 2 2 1 111 418 1273 742 +2302 2 2 1 111 419 743 1274 +2303 2 2 1 111 405 827 1224 +2304 2 2 1 111 373 1075 848 +2305 2 2 1 111 374 849 1076 +2306 2 2 1 111 398 1194 825 +2307 2 2 1 111 399 826 1195 +2308 2 2 1 111 329 858 1276 +2309 2 2 1 111 330 1275 857 +2310 2 2 1 111 250 1060 851 +2311 2 2 1 111 469 839 1245 +2312 2 2 1 111 470 1246 840 +2313 2 2 1 111 247 1062 866 +2314 2 2 1 111 10 753 1227 +2315 2 2 1 111 62 1228 754 +2316 2 2 1 111 427 1037 892 +2317 2 2 1 111 244 1129 990 +2318 2 2 1 111 243 991 1128 +2319 2 2 1 111 458 1165 1233 +2320 2 2 1 111 459 1234 1166 +2321 2 2 1 111 308 1239 691 +2322 2 2 1 111 212 821 1001 +2323 2 2 1 111 391 946 1145 +2324 2 2 1 111 297 742 1273 +2325 2 2 1 111 298 1274 743 +2326 2 2 1 111 213 1005 824 +2327 2 2 1 111 475 1196 766 +2328 2 2 1 111 476 767 1197 +2329 2 2 1 111 555 1151 1135 +2330 2 2 1 111 190 1229 755 +2331 2 2 1 111 154 783 1143 +2332 2 2 1 111 104 1145 946 +2333 2 2 1 111 453 1056 979 +2334 2 2 1 111 454 980 1057 +2335 2 2 1 111 447 1071 918 +2336 2 2 1 111 448 919 1072 +2337 2 2 1 111 329 776 1265 +2338 2 2 1 111 330 1266 777 +2339 2 2 1 111 477 943 1146 +2340 2 2 1 111 247 786 1062 +2341 2 2 1 111 224 1040 871 +2342 2 2 1 111 225 872 1041 +2343 2 2 1 111 413 1077 869 +2344 2 2 1 111 414 870 1078 +2345 2 2 1 111 495 830 1240 +2346 2 2 1 111 409 884 1098 +2347 2 2 1 111 410 1099 885 +2348 2 2 1 111 251 1204 1178 +2349 2 2 1 111 252 1179 1205 +2350 2 2 1 111 406 1037 962 +2351 2 2 1 111 223 1189 962 +2352 2 2 1 111 613 786 1225 +2353 2 2 1 111 372 920 1039 +2354 2 2 1 111 174 830 1126 +2355 2 2 1 111 179 1224 827 +2356 2 2 1 111 478 709 1178 +2357 2 2 1 111 479 1179 710 +2358 2 2 1 111 244 1154 1129 +2359 2 2 1 111 243 1128 1153 +2360 2 2 1 111 375 916 1079 +2361 2 2 1 111 246 1061 901 +2362 2 2 1 111 556 1161 1187 +2363 2 2 1 111 557 1188 1162 +2364 2 2 1 111 188 1187 1161 +2365 2 2 1 111 189 1162 1188 +2366 2 2 1 111 485 787 1147 +2367 2 2 1 111 486 1148 788 +2368 2 2 1 111 347 1098 884 +2369 2 2 1 111 348 885 1099 +2370 2 2 1 111 416 1252 852 +2371 2 2 1 111 417 853 1253 +2372 2 2 1 111 369 876 1027 +2373 2 2 1 111 396 938 1243 +2374 2 2 1 111 432 1255 1217 +2375 2 2 1 111 717 1240 1085 +2376 2 2 1 111 255 1048 1165 +2377 2 2 1 111 256 1166 1049 +2378 2 2 1 111 221 1108 879 +2379 2 2 1 111 222 880 1109 +2380 2 2 1 111 417 1230 811 +2381 2 2 1 111 503 1165 1048 +2382 2 2 1 111 504 1049 1166 +2383 2 2 1 111 525 1160 1184 +2384 2 2 1 111 262 825 1194 +2385 2 2 1 111 263 1195 826 +2386 2 2 1 111 508 1248 1075 +2387 2 2 1 111 509 1076 1249 +2388 2 2 1 111 177 1027 876 +2389 2 2 1 111 314 1226 876 +2390 2 2 1 111 245 943 1203 +2391 2 2 1 111 427 962 1037 +2392 2 2 1 111 316 1182 900 +2393 2 2 1 111 308 888 1239 +2394 2 2 1 111 247 1105 1225 +2395 2 2 1 111 480 648 1106 +2396 2 2 1 111 481 1107 649 +2397 2 2 1 111 315 899 1020 +2398 2 2 1 111 317 852 1252 +2399 2 2 1 111 318 1253 853 +2400 2 2 1 111 455 963 1160 +2401 2 2 1 111 260 890 1158 +2402 2 2 1 111 261 1159 891 +2403 2 2 1 111 736 1254 946 +2404 2 2 1 111 372 1039 998 +2405 2 2 1 111 436 975 1126 +2406 2 2 1 111 572 1242 1080 +2407 2 2 1 111 385 1280 765 +2408 2 2 1 111 169 1160 963 +2409 2 2 1 111 484 1125 1055 +2410 2 2 1 111 436 943 1130 +2411 2 2 1 111 464 1083 1173 +2412 2 2 1 111 465 1174 1084 +2413 2 2 1 111 436 1130 975 +2414 2 2 1 111 468 1244 921 +2415 2 2 1 111 221 1277 905 +2416 2 2 1 111 222 906 1278 +2417 2 2 1 111 184 1180 925 +2418 2 2 1 111 185 926 1181 +2419 2 2 1 111 250 1100 1060 +2420 2 2 1 111 236 1102 973 +2421 2 2 1 111 235 974 1101 +2422 2 2 1 111 469 1245 822 +2423 2 2 1 111 470 823 1246 +2424 2 2 1 111 431 973 1102 +2425 2 2 1 111 430 1101 974 +2426 2 2 1 111 384 1239 888 +2427 2 2 1 111 36 526 994 +2428 2 2 1 111 169 1184 1160 +2429 2 2 1 111 477 1203 943 +2430 2 2 1 111 500 1060 1100 +2431 2 2 1 111 436 1146 943 +2432 2 2 1 111 184 939 1180 +2433 2 2 1 111 185 1181 940 +2434 2 2 1 111 376 1183 818 +2435 2 2 1 111 396 1243 883 +2436 2 2 1 111 400 1256 1097 +2437 2 2 1 111 407 1129 1018 +2438 2 2 1 111 408 1019 1128 +2439 2 2 1 111 452 1018 1129 +2440 2 2 1 111 451 1128 1019 +2441 2 2 1 111 416 868 1209 +2442 2 2 1 111 525 1193 862 +2443 2 2 1 111 323 832 1279 +2444 2 2 1 111 430 1222 1030 +2445 2 2 1 111 431 1031 1223 +2446 2 2 1 111 401 1193 1184 +2447 2 2 1 111 241 1207 911 +2448 2 2 1 111 240 912 1208 +2449 2 2 1 111 20 1220 907 +2450 2 2 1 111 52 908 1221 +2451 2 2 1 111 245 1238 892 +2452 2 2 1 111 407 990 1129 +2453 2 2 1 111 408 1128 991 +2454 2 2 1 111 596 894 1250 +2455 2 2 1 111 595 1251 895 +2456 2 2 1 111 421 998 1039 +2457 2 2 1 111 560 1178 1204 +2458 2 2 1 111 561 1205 1179 +2459 2 2 1 111 240 932 1213 +2460 2 2 1 111 241 1214 933 +2461 2 2 1 111 364 1003 1261 +2462 2 2 1 111 363 1260 1002 +2463 2 2 1 111 489 1261 1003 +2464 2 2 1 111 488 1002 1260 +2465 2 2 1 111 345 902 1262 +2466 2 2 1 111 346 1263 903 +2467 2 2 1 111 35 1020 899 +2468 2 2 1 111 174 1126 975 +2469 2 2 1 111 281 1083 1136 +2470 2 2 1 111 282 1137 1084 +2471 2 2 1 111 37 900 1182 +2472 2 2 1 111 464 1136 1083 +2473 2 2 1 111 465 1084 1137 +2474 2 2 1 111 435 1055 1125 +2475 2 2 1 111 452 1129 1154 +2476 2 2 1 111 451 1153 1128 +2477 2 2 1 111 406 975 1130 +2478 2 2 1 111 525 1184 1193 +2479 2 2 1 111 430 974 1211 +2480 2 2 1 111 431 1212 973 +2481 2 2 1 111 363 932 1260 +2482 2 2 1 111 364 1261 933 +2483 2 2 1 111 329 1265 927 +2484 2 2 1 111 330 928 1266 +2485 2 2 1 111 526 1143 783 +2486 2 2 1 111 584 994 783 +2487 2 2 1 111 481 649 1218 +2488 2 2 1 111 480 1219 648 +2489 2 2 1 111 526 783 994 +2490 2 2 1 111 416 1119 1252 +2491 2 2 1 111 417 1253 1120 +2492 2 2 1 111 514 1097 1256 +2493 2 2 1 111 390 993 1251 +2494 2 2 1 111 389 1250 992 +2495 2 2 1 111 401 1017 1193 +2496 2 2 1 111 271 1233 988 +2497 2 2 1 111 272 989 1234 +2498 2 2 1 111 489 1207 1261 +2499 2 2 1 111 488 1260 1208 +2500 2 2 1 111 233 1173 1271 +2501 2 2 1 111 234 1272 1174 +2502 2 2 1 111 610 1252 1119 +2503 2 2 1 111 611 1120 1253 +2504 2 2 1 111 178 888 1231 +2505 2 2 1 111 462 1173 1083 +2506 2 2 1 111 463 1084 1174 +2507 2 2 1 111 484 1111 1151 +2508 2 2 1 111 191 1151 1111 +2509 2 2 1 111 18 1030 1222 +2510 2 2 1 111 54 1223 1031 +2511 2 2 1 111 529 1262 1058 +2512 2 2 1 111 530 1059 1263 +2513 2 2 1 111 105 946 1254 +2514 2 2 1 111 326 1085 1240 +2515 2 2 1 111 223 1021 1189 +2516 2 2 1 111 599 1258 1194 +2517 2 2 1 111 600 1195 1259 +2518 2 2 1 111 436 1198 1146 +2519 2 2 1 111 415 1225 1105 +2520 2 2 1 111 587 1117 1269 +2521 2 2 1 111 588 1270 1118 +2522 2 2 1 111 253 1269 1117 +2523 2 2 1 111 254 1118 1270 +2524 2 2 1 111 526 708 1143 +2525 2 2 1 111 264 1146 1198 +2526 2 2 1 111 462 1271 1173 +2527 2 2 1 111 463 1174 1272 +2528 2 2 1 111 262 1194 1258 +2529 2 2 1 111 263 1259 1195 +2530 2 2 1 111 36 994 1020 +2531 2 2 2 112 161 1295 1317 +2532 2 2 2 112 161 1318 1295 +2533 2 2 2 112 1285 1307 1312 +2534 2 2 2 112 1293 1312 1307 +2535 2 2 2 112 166 1320 1294 +2536 2 2 2 112 166 1294 1319 +2537 2 2 2 112 1290 1301 1315 +2538 2 2 2 112 1290 1313 1299 +2539 2 2 2 112 1284 1299 1313 +2540 2 2 2 112 1285 1314 1300 +2541 2 2 2 112 1291 1300 1314 +2542 2 2 2 112 1286 1315 1301 +2543 2 2 2 112 1292 1316 1309 +2544 2 2 2 112 1288 1309 1316 +2545 2 2 2 112 1287 1305 1295 +2546 2 2 2 112 1289 1294 1310 +2547 2 2 2 112 145 1299 1306 +2548 2 2 2 112 154 1307 1300 +2549 2 2 2 112 1289 1319 1294 +2550 2 2 2 112 161 162 1318 +2551 2 2 2 112 160 161 1317 +2552 2 2 2 112 1287 1295 1318 +2553 2 2 2 112 165 1320 166 +2554 2 2 2 112 166 1319 167 +2555 2 2 2 112 1282 1313 1315 +2556 2 2 2 112 1284 1306 1299 +2557 2 2 2 112 1285 1300 1307 +2558 2 2 2 112 1290 1315 1313 +2559 2 2 2 112 1284 1296 1311 +2560 2 2 2 112 1282 1314 1296 +2561 2 2 2 112 1282 1296 1313 +2562 2 2 2 112 145 1306 146 +2563 2 2 2 112 154 155 1307 +2564 2 2 2 112 1284 1313 1296 +2565 2 2 2 112 1283 1310 1316 +2566 2 2 2 112 153 154 1300 +2567 2 2 2 112 144 1299 145 +2568 2 2 2 112 142 1301 143 +2569 2 2 2 112 151 152 1302 +2570 2 2 2 112 1296 1314 1331 +2571 2 2 2 112 1285 1331 1314 +2572 2 2 2 112 1283 1311 1312 +2573 2 2 2 112 1321 1305 1336 +2574 2 2 2 112 1305 1287 1336 +2575 2 2 2 112 1286 1330 1297 +2576 2 2 2 112 1287 1298 1329 +2577 2 2 2 112 147 1309 148 +2578 2 2 2 112 156 157 1308 +2579 2 2 2 112 1286 1301 1330 +2580 2 2 2 112 1287 1329 1302 +2581 2 2 2 112 1282 1315 1305 +2582 2 2 2 112 1289 1304 1319 +2583 2 2 2 112 1288 1320 1303 +2584 2 2 2 112 1286 1297 1317 +2585 2 2 2 112 1287 1318 1298 +2586 2 2 2 112 1289 1332 1304 +2587 2 2 2 112 1288 1303 1333 +2588 2 2 2 112 1288 1333 1309 +2589 2 2 2 112 1289 1308 1332 +2590 2 2 2 112 1310 1328 1316 +2591 2 2 2 112 147 1325 1309 +2592 2 2 2 112 156 1308 1326 +2593 2 2 2 112 150 1329 1298 +2594 2 2 2 112 141 1297 1330 +2595 2 2 2 112 153 1300 1324 +2596 2 2 2 112 1290 1299 1323 +2597 2 2 2 112 1291 1324 1300 +2598 2 2 2 112 144 1323 1299 +2599 2 2 2 112 1289 1310 1335 +2600 2 2 2 112 1311 1331 1312 +2601 2 2 2 112 1292 1309 1325 +2602 2 2 2 112 1293 1326 1308 +2603 2 2 2 112 143 1301 1323 +2604 2 2 2 112 152 1324 1302 +2605 2 2 2 112 1290 1323 1301 +2606 2 2 2 112 1291 1302 1324 +2607 2 2 2 112 1310 1322 1335 +2608 2 2 2 112 1282 1321 1314 +2609 2 2 2 112 141 1339 1297 +2610 2 2 2 112 150 1298 1338 +2611 2 2 2 112 158 1304 1332 +2612 2 2 2 112 149 1333 1303 +2613 2 2 2 112 1291 1314 1321 +2614 2 2 2 112 1284 1327 1306 +2615 2 2 2 112 1292 1306 1327 +2616 2 2 2 112 1288 1328 1320 +2617 2 2 2 112 1294 1320 1328 +2618 2 2 2 112 1282 1305 1321 +2619 2 2 2 112 155 1326 1307 +2620 2 2 2 112 146 1306 1325 +2621 2 2 2 112 151 1302 1329 +2622 2 2 2 112 142 1330 1301 +2623 2 2 2 112 150 151 1329 +2624 2 2 2 112 141 1330 142 +2625 2 2 2 112 1293 1307 1326 +2626 2 2 2 112 1292 1325 1306 +2627 2 2 2 112 149 1303 1344 +2628 2 2 2 112 158 1345 1304 +2629 2 2 2 112 1305 1315 1334 +2630 2 2 2 112 1287 1302 1336 +2631 2 2 2 112 1286 1317 1334 +2632 2 2 2 112 1291 1336 1302 +2633 2 2 2 112 1304 1343 1319 +2634 2 2 2 112 1303 1320 1342 +2635 2 2 2 112 1295 1334 1317 +2636 2 2 2 112 157 158 1332 +2637 2 2 2 112 148 1333 149 +2638 2 2 2 112 1283 1312 1322 +2639 2 2 2 112 1293 1322 1312 +2640 2 2 2 112 1283 1322 1310 +2641 2 2 2 112 1295 1305 1334 +2642 2 2 2 112 148 1309 1333 +2643 2 2 2 112 157 1332 1308 +2644 2 2 2 112 1293 1308 1335 +2645 2 2 2 112 1289 1335 1308 +2646 2 2 2 112 1286 1334 1315 +2647 2 2 2 112 1284 1311 1327 +2648 2 2 2 112 1296 1331 1311 +2649 2 2 2 112 143 1323 144 +2650 2 2 2 112 152 153 1324 +2651 2 2 2 112 1297 1340 1317 +2652 2 2 2 112 1298 1318 1341 +2653 2 2 2 112 146 1325 147 +2654 2 2 2 112 155 156 1326 +2655 2 2 2 112 1283 1337 1311 +2656 2 2 2 112 1294 1328 1310 +2657 2 2 2 112 5 159 1339 +2658 2 2 2 112 7 1338 163 +2659 2 2 2 112 1288 1316 1328 +2660 2 2 2 112 1292 1337 1316 +2661 2 2 2 112 6 1344 164 +2662 2 2 2 112 8 168 1345 +2663 2 2 2 112 5 1339 141 +2664 2 2 2 112 7 150 1338 +2665 2 2 2 112 1285 1312 1331 +2666 2 2 2 112 6 149 1344 +2667 2 2 2 112 8 1345 158 +2668 2 2 2 112 164 1342 165 +2669 2 2 2 112 167 1343 168 +2670 2 2 2 112 162 163 1341 +2671 2 2 2 112 159 160 1340 +2672 2 2 2 112 1283 1316 1337 +2673 2 2 2 112 1304 1345 1343 +2674 2 2 2 112 1303 1342 1344 +2675 2 2 2 112 168 1343 1345 +2676 2 2 2 112 164 1344 1342 +2677 2 2 2 112 1291 1321 1336 +2678 2 2 2 112 160 1317 1340 +2679 2 2 2 112 162 1341 1318 +2680 2 2 2 112 1292 1327 1337 +2681 2 2 2 112 1298 1341 1338 +2682 2 2 2 112 1297 1339 1340 +2683 2 2 2 112 163 1338 1341 +2684 2 2 2 112 159 1340 1339 +2685 2 2 2 112 165 1342 1320 +2686 2 2 2 112 167 1319 1343 +2687 2 2 2 112 1293 1335 1322 +2688 2 2 2 112 1311 1337 1327 +$EndElements diff --git a/examples/headland_inversion/images/seabed_classification.png b/examples/headland_inversion/images/seabed_classification.png new file mode 100644 index 0000000000000000000000000000000000000000..7abf92410515b4efd2991054533f4a5ac12fb14d GIT binary patch literal 45932 zcmY(r19)BE6E>P6Y2&7`n>4nKMh#EW*tTu9v2EM7ZQJIFZQPyT|GVFF@ACvZ=Ok;- znlS190>J*!C@;2X4+h_WpN1VZ=w z=ZAjF0zC+b9}uE~d~!}H$E%J`D7&{E7iTf%;1B%JAHIJLu>12_I!KQ1k6cBtQhDo9 z?TTybok|%bXYa>m}u77Ev^nZCkNTFu|+OPcl(YWkO3+ zCgufdV(qZ(_!BGu&Rh5YNqeD5|t4p%0WY0c7 zyOrT^ez)YaSojyrV*lc;(ZAmI{Cae|9KY@f@_cfL$k6PkGl!e*qN9?S?xUl!Qx}?+ zvr~tCX*@gof|nGQSICBgAScf^Z*QlbFBF`|mJ(RQlliGsMSrSFUmT0Y3F%B8{Or!Ur;H54ncF$EwYBvwjpK|M zQu=uRDYt9lm9zH3KV(49U7KUg%u(c9VdCO%$;Qe`Xcz_u+DTvDER?@7b=2^WC(m{2 zYe8bybr#_pGCb*`C)XvTJW#3V=R__xQ74*n0Q;a+sWlN&QCYh180L>L9*(2d)7OuT ztqrp`jf|RvT2|d9u+#m{c*VL<`EZ@- zX2VxQ$ll`9#`>LCm(D_iJ2iHFss8lTBGvQMXrDPt^X=oH*lfY%u?cU2my3 z{rKQ^F94V9NLI40CQ=;IS7S08G{Ta4Fj2|)@^n|;1|d|+WVyx_N2@y2#5}Cj&J$b$ z&uO&C6^91O5X$Cn%2y6KL;x3VOnCQmxB0ev>n!IG&#z1w13t55L_|*L^Np}b-Jx)F zCJ~F9Go}o!&;kKhg%y#TzZ(XDg?PQe92*crcly?s>Y37)*1&$J&`^Wtls@RdZvJFG zs^Cquq%GLP+`Q}nUChIawJQ>r$=UDsIGnKw6|i#CU`z|J->jvBEA3go?euT#N!M}* zj^EYjSN8fM7wya+nbn)^HvG@fe{(QNds9E3s&Y7*2;c3k+iVY*m0?&+*Dt^Nh>TV_ zM2prN$Pm9v+a&t#*@ALcP8%T}9NTLlc0{|_E8RMK(>xj;uPm=u6Woyq!ZW|#9{W=z zFg?FiYks9Wk4cI@;G97&7w~7=Z`&&dWD>lN5=i{UA5h>p*h>N%KmW|&Q4gEe#vGnE zrhLtFq8CT~^mB?J5K9X@wa$JrW0BK8wpXZnNimCa@1%MjJTy4^6W90Mp6sQq7Epl& zhvrKujhaToJv$1e>5%&d+W1@?2Y3HI^-mR_HLlo?^Zvw*3W&&0yGLw)J|~}kO_Z|X z&&g(IS*L+sl@}L}%4B@kRJvRc09ys9H0^L2PgE`lrZ>o&yHA&sSL?b95j-WD?L2@` z+hmlK!35rt-Rf+qfpd$|_8MS>IB}%W9)B-PaP2mue{iaZD;~_a(>h*%MkqWu0F%A99$3 z>;zHo?wJl2Mgl2*8UoNe^F0a$30Too&82eCaX9O4;0#1oAN>>qyn}**8n&vglv)oUd0et0kW~aZvW8U8DGy+nKvC1e zfR1YYac18IqlniuU^G{_X8qf;o%Qi|dR&#{8LbBV(m@*46+}eI34zPqw zFnGL8##fej58uDQ2=&G|DNW8h$^=kg+VhkwY5nF1sW@00+bf3tu;=;sm6q%351-Z2`4)jq^mvs%Pm))kVK^M8uRhCxLl9Ch~*L#Iqpvr zNcZjGPDpHw zFWh*Kh6HcUPHMpJgfGUwob^cGv)_fY`$;&*I2|(|#E`%%2(4Dp3-9iaa&Weo1PY2| zE$7)HdzTfG3=ur_%=FUWV5vD*wev4O7&S(hK2Tr|lI+(Y;yJC=Y^1KMqkPtqVCB6i zn<}sUt4x_%vR=&4r0bOz7=?+Hl|{5lD+u{=T|MTG*rSC76V5Hu!)b|1_o7Za&Mhm_ zj0{?P=0zWG|I;TO{71e&L<%>jzJiQO=H4VNd&S(j6=uBspv)?ye)Vv>^C`ITY)z*O z=P2KnG#YaKK+RK@3V1v25J~z--6nFdi`QhH;52U%DtV4-hjhyz7B%(7j%W9|mP zw1TDRZy)Y&Swo|9JBtbgJXVVsx1~s3gXAQ|7%}Ep2Apv}UXL?b8J(X%eW_MpuPkm* zJtmh~Oe&Y5?K99V^Sg_n&{nU-Tp8IHo}krgau zz=286<#bPifWh`-V5J-fj+;>=TsG|Gv#XY;x66$f9!9;T;{7@@-dg%85q+n@F)C_v zFo7Twcm*?sCfT>W8>~)<3%cjBQK$K7(e{_Poi^N#Z0oB<+XK{Ii_^~8P0_t+e&2|% zq=LB|y@c;xDCp51b}GA7ddtGTVe}0WXG%+chCua79?fDx^l~A@4E+Z<;;L~^sm9Xi zyw`;E=(gZFv2;39dN(>_*ult%0?ruu{oJXRGbf9CJ8zf+>kpz#D)f@anpP=Fv2nQC zuQ1Q;<0JtiQVc$PSVPL9iO5d=hFW77{;$k?Se*mAyog0sgIe#gc})fy$t&EDEikwm zyvR7P(aEP!Hlx&hI{;5W+er4yq^P&SkZbq$2Rj-lY2o>9y_bk+q2u@!6hWW2A^-SH-M3iKqGE$o-V5Y(hQsmZ}IQJyPJNKloY$Nx40@rl8^rbzKY zd+-CY!m*sF6pJy6;WXKpPOl{VQ3dGML_6Z3?tx?Y?^(q2a2Gs>5TEadM81^CbxpN^ zfXJUI-%w{Vz5i-=Y)d%}43eiRO@v!9wCMT|h4X^pFj0EIKH7#*5chg{R{_g1^2x5J zmzCzpU8d7Rcu}%QqM|JOVQ?FFlvIyGVQE z6#ZDXtTROolEf5R)mnm4y_AdMVxf)-PdN{;{?*?{f+xJ&KVGs}icE*f!31NTFQ6O! z&#TL?mH3X;AIe+h3por0)d~idODnOI{)1#{SSToobg%-Gpol-rieGL*UJ<~gPOUY! zFF$|x+0NEhwmcUFr=x>+$Iz|@CBD?bcaOuwvgYM2rMN|Prm%Ae^$M@8k*~r-!NXmS zH6zTG6+hA}GyJsMSgBEtBFu8IWz#zU5lXrqU`MohcW&>zVl!L{gr#HMBofDUcl5V5 z4%Xu%jat#|SWFrhd$`Mk8{ZmZye20Zc>v!xDzRZ2U$S(9+|JX<>5rjpQ8%(>Gi+@Y zYZ1ngvPcH+$P>_rnyHJfbS*!5nkfd_FV!rt;xvPTe9M-omZdp56XT2~3j+r)aj~#% z<@QQ#zwK9nQ>CAx-uQ7WfzX=uPlt;IxgY&v;inKB9*SN+LM-c=o%Q}IJsd7xKeANg z;5e-_20WBNtB6W{oze8(hPwZzAtV`v!u?_KPsLz}6^G#T1PrGwa;ZRaC(571#C99h zG;xe-(ogCRfxUObCWar+J?lF_^WlMZ82={DHY=06Jo8zMI;zilCR!?c*ZWa2e!8qvyy znCvLcz6$X4H2jttOM01Alxztd8DoHK^owjjQ52B5$7U2w)Q5MrzeTb1U3#m^z zT(`Q{btThYCt5R*igekSBwFi&dRGmo zX{zKk-ag^{ppWkEs@T?jA4QzG+NM7|#0rH>m=w~8vv&V0Uuu3yQRlfM!Od^wkroOO zPfS8W0?1@+FR!kgFE51epV>7R=38~chu%gJ;FFD)w+`0li*%+N?IYNE*`Yt9rpO=I zR;_W2*oIM-R9d>{;XwHbBK4g7o3x zO*}B&VNSAT!@quuk~HwBr)sHg0?hi`rq+ro{SGS5%aFu;N9MATHbuT>eQ%B`W8rrs z``|qO;YSkR55B02!fl?%tf+Z!n|h$0He=z)!MzC=@F$)rg?5>a4H{CN|rEvF&L>T+ti0 z2f^eSAF4GEQ&ha!8IBG9OIGV4=Jv?1| zX4nd56Gp2e{IK1NtKH@N^JtqVQiOsv2WbWwh37B7L(Ly@$4BABwQo(ns9w#&u`9NIZX`uk@+B3XysP%r|GS|SBu zA+eOsvd#NJlasA9o^M|giYL8mzZ#dx(qOW2I&wID91m}2&>BnAqmc13<7!csvkGtT|@?4ZGP=q6B@eZ2EprHTBo8P0RU5Os;~nX^ zSFNQ#?5{n+c|)ZOiHV5;7JK7xywJI$n6}^D4_j={Ny^0PbU{PQ*k*n4xKESjvE02P zBx%u_kzf!i<`EZPGPzg*+Wt!!)KzujD=tTKY#>lO)=nTe8uQ(h@dx98qA9M#Tes*_ z9~x+jS|^poH5lb}Av0RFWP*G2M>XG|`=n*Z+47L2K@Q2nXNduFg@dV{U$R4EHhpYn z42Ov&>)Da}vLbfk_-y|x=nFDj&u%B1YI{R45z?H7)VJ1UWX#Ii$Hoh;UW~9t(z0d$ zWLWj?zS`t7;;_*Tz;3I%F-$wpu6kC&jNf zly8y5T!mpL1|b9;dztR(mp;w;1(SDmGz%5M!igaa_RC{~(H{tZ3uGp(DStc_W@a$1 z-x|BECscl(h_}3SeyypI%((8=Fj^|FV*wR!j>%8v&Mc|l5pg$ScQ@ylFtBs+?%Rjg zcHYPC0-GF6G;eMty1Kz7+*qEcGY7L-w+J6lvq=7!J&E`x5G*b(hTw50u%|1Yt$riO zl9sOQDpQB?%bcjl6jCcuCw$Px&{BqJ!V{B>A^isN$BNC23RXgvZ~lZKUki>Je`z9f z__MdxGO?6Sxc)N#4CGmB`T67%g)Cl4D`|W*Pf+Gy;Jo!{Evqm`jLK22=y+tcHpP4z z?&}qyCWHkh`uJWd1+Hr1ii(s{BH&`7#>ir_C|A@iMZE}1 zaJ~~<6HZRPY3COwAM*#%XX0o`b&aMrq}%%Z!TXH{8i@encdBKzi@*se0Aj^T1L>@t zF%gu^A_Q!a94P&zcwMqpz7s-NkV>@74<~4QD40(*Yz8`MysZdLe8p8mfpl@CPk7e) z@KX}8!ajbebz9eY+ZG-8$%mNWBM^%Nx{WD_oXrmVao7nyW{^8HBHC--%ujwN{D#$# zNA6IF#~0M~*B+XnOCX96D7OzkgS@N#z+5cMQ;~Az!sWCZc}M+%93d*;J?#F=f%VH` zdu#+;8r-ez+;&&+#MH(KyG5BRSJ=7&KF$TsQC*|69sT&4#;@N+qBb8xxGA}&Kfc|` z`&xhjQ2>5nl|U2;sL1X6deM`fYqA{h9~opeL&{v8Ul-rrx4fHZ9Zr|yiZF_epMH|d zT;ued$A8*b67Y}E_ldOcawnO7#|3c>kB5F$?|BuH%h{HIUfZOn!78vz)L2HF3?PS3 z3sw#NfQ|GJaM*b*C?&j!=lm?xP=up2=pI@uQLU!|f;~*JLaEO1QlpxKfm0EESZr2D z$0=VP*MYvsIK&kiXTx5llfm0g&(ge!;QWtLZ707`w+G|~d*ccU;06|v@8n_thyKX! z`QGso@52)Kr2ew2Ivl9)RW^I~@v^1*OOcj6_)A7p(f$2~b6Dt{Z*dVF+cAdoa5uR_ zU9T8Z(=d0LB;M&klguSg1wd?moUP|8Gd$-K2Ua!TiJ9*5(Z|%15-NZ%MFm@B%_T9L zF4SnV=WC+9+5^}{;Z{Rmrm#^N41)6*3Bg$3Zq0*j>)2K-f(p~EKXsbD zC#Kqz=wskJ%cS}r|Cc2xEstRxHk)UlU=BxJ4U6o6)bgmZA%MyRROd0bF> z_dlX@h8=))n`vqafQX7QVXp8WjZqH1KPqoGnk!Mx=?>(;W--&6O>e&phb=OEF;`7+ zyK4qTN@YF;exSiWB;_Gt07;BAZUX*~V^7DOn{qaKOqoSZdX}^W`mk%#~^vMikKt zmOC|to^4jt??;u+TOdS4i6H&uvTetp+dit)G%%3#0Bf@TkEl##I&@R2Cg74yoU?nk zI6yW++~0DEOq<4iaMWBD%STIn8?jF?4FEsKI|o*~bK*L40~0EZa{r}FmqP1X4h){L*L-0N`-R~MB-uKbimX{xhQSblt8HR&`Vtm?1V;6KYEPLy|IEHUqI zV>#XS`63g7r{v;WH^De~sq@Cj<-kzA+M1(tSHX(C6a3L1!NSpqeV6R#N1DcgoxWrf z&u)1y$}t$a&0g#0tIU2vT(MAA;98A-1pW5r@e_=szx&L%{?%_Zsa2T{#dbF52VHc> z)^Ee??$0Yu9{0K`4T8H{(Pm=f&azP;BbAK|V_g*(KS)1gT_u)0$h=Z5Wfm5gJpJqp zEu_S}%1Eo0r;ZsW`{$Ejj<*j5M{ggP9q)6sah4xab(VKML(i6KgRg|s{_6N!F0sCY zOL$R{lEQ8@BKaT%=k04oNW-!Ar0zCAenF(zbd4WMBd3mw7k_9I< zm^w)KBfj81wAAHacefJ-8VhSKq4d`*tXJsIB5rPOKxS5HV{41ak%P#XP6Kn~(kcWq z&xIClF7O*UIwX&TT9%qY2BgANXT>qZgF8(5$LR){6|)d+KV7=kw(_KTGaj|VjM{9a ztV$PADK3=v>$fG(A2n}2Q9oGf;N#$rpKRY=v&mt|Sdxl-JQPq-ZZ^i@qfl=5W?%IH zayKAJe0F8z`5F6gmhvs7Z_ z6hJ(|;3tg`?LEUQPJR6&&IHfx`;1cX=NGJ! zm$0aP>*`+GOJ)ms)wi^<_Nq(@<}(kcj$hVaPT9_rFzP|Wc_eRbc?tIkLGtP=_YIPn zh>~Efh@_99M*{pfK7K)>kp2J=Ri-nk{Kx(Op+#z{^}TxRE=I(%WB)+&lD~aGTe8`Q zP@l-QxhUc<{S%aOPNnOwDAkpr@vm1s9;MPHx-WQo)k$rFrh6lW=fi1JPJ)HGcSGuV zJ9cox)t1f}c$<`QG|L>ex(hn;>TGt<2enwE<>XI8kI~_voUcaf6L`s$%$E-qJ2#HkKJdJZ ziS{c%c}qf@o1SQ5rF-meHQBn^ab}t<^FB$2+CC+xOl6j_zP{sl*r@clABqqG22;ZG z#l=_Q_u$bub__{c>nofmc*pHqX=o~eK|4$6pTzd6+!t2xdZOkU7Jf}aLMwS!;hZ02 z+-ZB}Vw7lNOXR!AJrSLcu3Q$pu#xLnfay1DXTf)ta$^vl9mM}W?F4$R~Q8f{m{an9`WzDyPvfW zjh4NeVx;!xRm4hhKwWvx`CTR<@)HQ`2rhe)$;)YBYnh}$AL9`n&!=+Ge*(}(r~Paj z!o7cg6D(?}b6Ice4!ALceOz+2>yz>7NXZn3rKienop>()dz@O0*8IAyq1brZUSV(R zC7Zl!Wpk*(1met_S3C%znKPc^A@=7>bUi<%(YY_R0()ZxE6Pzgyc<4+y%FRm$w?OxL(@(P$E&nS-T0l@pKQsxa$UhGVhq5bC(!bg$J0Ybgz$F5S$BJXj={NhV<| zH5pYubLPtzCJnuwp(uTCmIqJ_S8uFGGo3nP9e3$K>7V8abaUyXqBc+dDO^@dGKp3m zAC6#Y4y?Uwykk#Bax9Q|D`GJHFO-j)qeDK}ruIh70@2n6heM59X8+}GEzFJ#>E?FX z^0Hn!i~?SreZ^n1xnXmV?MXCg{{Ni?$Q!e{n*4}Pe>x?KDD=0PhCB=K0UtyzvsL*< z2oDtm%6ZigtRLM5@BJ9~D-S={GzI5f?qogYbOLjS>7umyseYT@x)U2y>gwuhcGH7j zF3gK+r$JS)DJA2f{f1<4XYh_{Dve(6K4BDD3u)mjwtKZ-Y=|{~SjUO}dMn7V?sg}h zBM_J{EqT&c3YVww z2ZUA~0WGNY5}PwrG>}XYR^z)3RkyxCKG(=~W~Dfj7I6yM$0QjJuhx*w-FW*-%}iv?cT_{ z8ZRF9( z9#wS(BM*f-?jQ>n8AB~Z`S0%uyGiZES)tk+U5SSIRH-Ne_tY&8Gl2|}HgB-F9X3D< zZWNbSS=(}P_j4Gn%WS_=f*6v0T}|IZhMu}@Sm(i$+?<&Y3pg??Z|isZ=58!GErZ@b zVkE+DLo$m#7i;=9{PH7@%0|ygO`}Cl`AqvuRaYmnWfS5xdg2W?92@((1EHaD7-fQd*~~YO$JS z?#;Et;swuMsrG`;NOSM97g5-*C^H}(e!&~05O&w%1; zEZ2FaHgQN(;typ8gCPv@6*^YzaM-D0`Qg$%s;{c5R8Zq4!H zA3ifi!D1#f`lATrNr7tFuMA=b@$ti{VS$a!o~7`xs5(F`)^G!*(ir5+RU(JJTTVAy zSzas4U)aZeUvqb7NSFs-n9##mi2%3T*693KT0g%*wCyt*J%HFzU&eTTN+L`_NM2X) z&x0)y;ny1_S*Yyasi9$jEPt12bNj)nKN{xD-5?q*y1!#&HHTE*lmBy=tc+g!%xB)# z={qZ(7aRi*^T{&~C#R}wTp$-kdU*|iquC5=*z&n5G^4;0l5#)&3)|oF;r+VEr z9Ajg5I|{M(p4Ps4xI<_Wg87@#g`M8)wCRorl9lLNRT0^K{4(sJE}7h;6cAkKq_Z7a zh_!!Bm?{>Oav+{xO~ zzMrf~M7=_Ugztu6@*BP)@w~0G3`ayX%#I1wv9sexxs6%%LR^+Gy0jl%)~z5=I*V{%#gxOZkMjF25DFDu-sHNx|(paS6#n&hDgl zFa3TtBK@$(8XJGg3S_dr51*{&aA`a=9}VAy!sSdESb^rVqi#8>HBYz*4PJaQh{T-W z9vI;HE5rR8qOWy#^NtxI@0Af>7}7#o{iWjfKC&59%JfV$+RG6b ze~c6gw%HJxCmcgRxy4BLfIRym6FuF|gj{jaeEr?;8bGlPKMvyL4uc1j(jceXXBOIq zGwi(~z@i#8COFMDC9;=u0UVm3+QE{2;_9^c_DimnJ2$w>N-W*6DhzJAsYz%+&HZ~@ z1ZdGb6=Y=p8WI5Rh@OTtWl`Ml@#&2I@gM9@T*+GJyVQEjwfVngzU|~?Ssjfp&ne60 z&sFQb;)K@>6H7LyT=UaK0eYykFj%gAhFJ4DBz?tGJ*>eF{!GPu>_y{;Sj z0{>%4@xw2olrth1^3N2bP##7*`NJBdm7Ra4I*tA7Q1_mX=o4qV`u#C`bJOOjeH9+C z49#pOEt|!^6Kypl+%b-56l~ok>&=bl+a^z?kBSr_iAD-Ai&Nz(>(DPkO6H9Q?B^n) z_}%$&(uCUS0SDYOsm>^~+jar7eHPl5$oKn(c4xgDR5U9ZX)H^6(cX@Vdv|g=1h205 zj`)SuE7wP8dLrXN#0Gc?8Mb4XEZ^MIsR_dB!x@8|IjaMv9oX0%v;9egGG}`B*t{K8 zQLnUzbyDKG9#xyL{kc5X241Me4adj`v@;|0M&<8GP8L4tRaDXeTw~k=) zhv)htG%|FTBets+kaREIw;^b2DR++Kc<*RMe~d-+1Qg&|x+CzuWp2z-Gep~p2Ew0B z%vu#%i$=a+^+6h8)cbR#k9GB4xLw!&E+2!DS|M~4evgi7z^OVt2vku02{+}uDiXK+ zc(dL5eBORD@r)pMTp7N&aM6kwxc$@SNa%H-Rb&N$Ev?q2sO#N58X@u z2+pFhA?Y7VR@?*0tnjDyA@kvFaXIgrFyohqen1uf8)^;3GfGmcL^|kstch28Ww*4?CdS3;;tpqN zh(=#6L_h9$dAVT;-QU7P5Y~HtCl^;!oS#cCkK}|QB@QJtyzNx|Xd|q*} zf;}xw7q`Z;(E2TwByp4wH}yS8FD+EdpSnqIHgaa4g}ojyUM8^Y_NpYJB(m%Z{Z5y@ zF?Mo11ya!3i-xe1-yn^@LlJ2o;AA&*>i8s2wB*PLK3a`mVeMakdbjn39nKDGpH-oH z6pecxaL0Ih9ogQCLZ;2D;u6}qm@3AdEFoAUwH6u*nFFPQ_;zE6%dd<0NC)FF9@|Uw z1m?a``T*fxfOP-U`I_QQLi(Yh*fRfXHQVMd9m0Nn#=_M2b*#_QUy0S^u*X(6v}at# z|LzRVu=u*IFN`Del3qBW(=Mh>+{$DBF1@qrd>jg&v{SNrSi+bif>M^@9VpWCu489@ zt@<-#mF(D&%qY)dR%;Zym2c^X9*>9nQl7OpeKtkCG&i1Y?!FL#JUshmJ^e#@`%xOr z+eMIr1Hgx^jN<{?c>G&r}9Blk(P!sh6n(P0ow%$6U_lrbJ)R=Y2(y#+m|6F88@tI z6_l&;_3uV18JSDDjOi@r?PU*UcL^&m!d$Bk>=&kJ>3Q*ZW7&%^e5fRKv>QRS^mZ%T zKl+WYB=X(z>M8cFQ)y&iDc4H{Yd4Qa@zTUDFU7@v0iE0_4;85N5ye5uih&Z%fr>ic zNQAPTSAEq#Hi?;49&3 zE|b&!wRk?it*3yoMP4I3v};Zs3!mCb2PZ!(lrwD-V~zBCDVA@5hGJ<@rg_*HYoUh) z7k9j>>n=mCIRP)rWqeXBao-=2uRc*2(0s9v9;p7Me|9P`Z{w4T6?&siXD6wBA3b6x zkGK~)E?qL)eqHdzCb(yRih9nRkC%t5%i)W-xq$k~(g<#x4xEN*y^X4^WTKWyOw^G0 z1KnHaIUu!ePUCWH2LLnhGP|j*(K7aS5f6^Gr;L8SD#e6guR9n1sgulZ>$v%p&Lzfz z2`gB1GXC3NGEmTQe9}g~<0GDANqYBxwHWN1yF3;o5J&=AipNGt2W*MB~agJi&oa-}yd(Tv&p1 zS@r$8Lc_bZa#>EWk8>S^xtRia>xKrCiWFuwoz^dCT>*(i*{Wpfbq=+__xLN4wSSj3c}+CI8edg7t6yxyA13dK9IP zT9brRi&0Gm+*t9l!kv2z6SDLtFZh?#GdT46gh39zPOS?EKBX<6)EU_g{7LuG8Z2e+ zV)L~)ZR-)^ZkNjnyxc|Ypu{gKS=F1QA{Qy&)6fZUKBmB)n9iZ9ZQ+AV0md@Yve`mK;2v58C|qLH-DAP3Dh~ z{rEJvlvSk%x>?hFG*0Yge=!*k%>Nb{vn2~|AMO3wPBi6QA8ii}b@V+<4XK`seE8rqAAMCY}C6W+Xm#gSHUWi486h zl$;zkOP}KDKe}G@rt2lyzKY}&_iE)_6#VqzYQj|mg`j@pK?GYjN+fGDZn!wzmi`AUS!EawNu4dd zlkq4Y8nnFP@77$1r?lk;n=M<1AYY2tv~H~M&{LLa2g-}><}6FonoF)cE5VE8O4LVa zSuD$&l?hXI5v(fcxCI?-asZ~5dKkg1#;rv?S#CsBG6_AogyC@5hGs5}SnMs=n}N;0 z3}FUO8%vROKL|wsDs)22|13g1B|>4Es~vY#<=j#n57=n%yt4=!!ZOO_X6g9mTtCos zH=O1mkN2Ryo1mh1i;ZKJFFhc)q5A)(DbPTgA~udd@$*SJWsdm2l(V2zMmEE|sVfH4 z>So|Q<2+%=XwYJ8e|@ARO|=8Sv#>%a7>&;go3y6jnFA&Wwd$@&iA4$R%UcZ!K$^;- zt|cCAz0wO6Xt$IB36IgNM6QR<357IT7K0f|-4nA7JSwcn6ey`dA}l8CxT?RikW>A6 zVPO1)`;y(g`RZ4CSSmfqI@4`6`Pxoa3C8z3;?LhdWqN8x9Fo#!L@5;5=#)kKhJI?9 zEsCgS>5tfFdML;ZN7G1+6@{XV_I^e+oT~ZrlODt61%1>lrEXg2Xk{v1Iq`u5NbwS* z_kJ#xPjN4crcG~Uhv;0Upv{+u(AKw$>bpfB8p_)k1w=?yFPv5?rF`%CV?MULq1|I3 z<-|&sHHL4xncPd^SyugjvzZg48jq4g37~cptmi|&(|Ep-ud@i>%oL@eLRm5_QezVN zP6WVFm*oguw6NxMgNxp^`bHQ#V|XAfPiKtr8PK1BNj~q|4q<1kpDM36cQR9fYcHdC zvjZ;^Bi!!w+q~DhmMCd>4ApJ(4TmWQPc2`OBt678mvIo=ozzj1i78LjDC|P-Cd)PG zEB*Dq%VD1?p$R{=a$HS6x6C%(|LP4G_EC61I90~#CijQKW+#x!zp@||$NsDot!K4d z#Rg?zUB$CD+QP)E$#S(#kK*^byVpg92t!vaM33k&zCCLh4X*9(JZ1{DlzDlcA?w)Y zCCysS1bfs9s8p}DQH9+F2PdW~jBu6)O_=%Ce2n#cjr+CF{iT11y=1}^F+cO0E?ErY z;TMQ}RF%_wzc0lyOsbh}QoAAMG*A}ylN9Y^>M%j>uS&4Co^p#7v%kCZlw}Cq8H23B zlZ#(Ui%^#9hF5c|{x5Ij)}q64+TszK3W-;@*Gy;bi%?3;_dZwN^7~DFGcMTd(Js`E z2=%VMANk?#EpM_?w_n8HX2j#|#Szc-z=UKiS`lo&2U(^pNmwQoMMo@6K>zwt zS>CMeesZuY7r_pm2IF8ms`CSmaI+41e~S>EcGFO|G@EJUx6P{2@+;5f?eipxQ$5yG zBGM#TVQI4vFQFf_zb@8vQi4WvLtzKbahwYhbASEq=C4N8`6t#`-H&p_LL*BqU8!Bb z0*N5O``P?>f7X13gzfkMM~6IXcr!2}FOUbT{~^G3ZmlTd|Hn)b zk=rEv^BO}aSN)$^nanw!cecRrRHoV3GeG<{ao`!$TbNp8y#!5QQi1y+HNKuGgN=DBxqmx= zx_Q~EVxgmXF&NB*2Uw!!QPQy4 z(dFpfcSlbbE3nZ&afhT=-L{n;(;F&=1Y+TkfsROndIXALCXTMJ>%>?1(!vnczCtFD zG{&UjV_-lN_(_em`mQHpg#yX&xe|nGih_Nt9^G9S$m2IhTeSDpx4`+(MQ9*WBv&Ls z6!JekG+9LqHaZILFI+H6`VFSP4)uj%i%Lm_+p!7901y25*$*Gsitf%w0<(6HiT>rD zvln0Z+spfI%*T$whI;D|c#02>S(M4&J}5OUZ3Q94dbWO>JVh_ZR{VFBfJ_L{B;22R zY#^F88sQ>TF!{htDd;OAByukzlk=VEzr8bj-`=<`!`fb>YjBkS8p934<5XHxKGs|p zGxvZ~2qO3M_U;N?QocNfF3X=Le-jT=%vJXjeGv>MOz(=C#hT`^z3L5@e6_bnN;KXq z-*IwF>KZ&@=CU`V5G1%AW|h1BEVqT-Nb#&*!-VX2dD9ork=fQZL{-JzU z87=gk(=NzS;L0Zy3?&W(LWO7~-TzE7n>`aUhp0OQlCDbgT@jRisEo#4Y_CrP(~$_d zF@f9t)(;b1Vy$y^o8bD@`;Qdi>vF$bqN8Gekc+agrk!OAh(`ll^X4Ww9trP3ulq6T z>+Yi#9+QZI(Ht4q5i%hb`O{@$teEMkX>W)w;xNz@O3z_TV&gsXP~ z3#3NAWS#982wDc)o~8|(J053xygWuC?0W&dKQxbeFV6G=HOR4gaU3d@&5bsCq1^9; z{)f}d1AZqSYiZj$otBbg$Cw3X1H>&)xns_svd*r42(l4#@;{Y>S-pbvaX_9%?V!2r zau}(v5rQH_@+~YNxrNuyqU#~tDN%9a9u7!|pepulj)qg}jeP!jDdkmF{dk!p3AgZpTZJvDlxL)%&)teb>_WDP97Pi}|To?l~Au)axY4}H-> zd-qbDM}uRMYK^}{f?>#jvySEb0~#%#I})l<(p(j{bDy%#eDp&eIlAbU<8qfPl39E#mvA` z7SgugD=Gd1O?3G5n0x!vDqp4)c|DFNz~8SC`ex?isTs_5MG^Nk zAz0%j10(TMGE_4RW8bQ6<LG+at`|@WI-UA7Et^gRHCs#BYNJGuJPJJmaM@dDU$g7 z;d8BQuD)!r1ExOhRe#e2gJG(}`J3%2A?d4t2V@b<1wD%~Jhci_<&e7+s`ciNfWy2Is>tU-$$zd*Hlx z?yKTooEwp+vsP|iT2@4D{(+MBGAUA?`D1sLiQ7Hx;B~msjNw7wx5dtMlO2uB2Iq_x zXELl`s_J!@GY?ZUGOfg(Aet0Tm%Hnl#fiZ230BOboF}rCwD>@2}lhZT`}1 z9Q1Da=AFxSPE0(w73zr| z(^v|Oj{-(S#M7rUVKl0Gyfd7B{arM2Fx{^RYDk8*Q%BB|cmI}>chi_$8%x{%)Vob9A43dLeH3(bM)ybm5nRt}ZCYU)3trxQdjy8f zPl&wK^NkmSPQMY)ZNcAoC56H3qeG!fZNW1|=_5RJeEPo-A-{SKz74^1SdNBe4&+bJ zEfUV@N*^mOHY)CBKUg_v;=4CJ4!fPqnZ-Kp`Lw0DI9}MEVzx2F1CJW`|N5q231hPx zki`)Hu%pUKnuq-*iqQ2^jwiu!=PW%$E?PSub^0*EIDA{z+`wa>fP8GLY`8_Age$JZ z;e0FwL-Ne#+Gk*_l>)fA_VC!mKY9!7VW@OXKIs)ZOLVOn2>d3r$CrH<0s1@eVfi1K zo$^$yvk~alc@I+Kl-2zWAy=7}QN`dD45$^>IaPx$-EEfCLPlMqd4_>P-0tc2ygb>8 z#)iuH_&SoPY7<&(yR>`+|tBEDU^NR#H?-+f6H z3UpU#vdaWnJd3y>v5kJ1siqm|F ziB+j(YVdr`1IJ}CUABJMV3Fg?aJ3zWT!zbDNc2mJy^0Y&?KS!F;H*-s^8S%#ip2Ae zFzHW0h(sTwwyA$#)9iUWa<75Y>*4!h z^(AF)QA#HzG|Q)mmFI1(JBv3J>m@dOt;zEh_^zMJo!n?({tWHw5j(7bminK%(cwAE8DXCmvYee zP)-_xQEPJ?@!L66!C6HE)2MimfJ(4WLiUq{bA{JfslWx$YQA1|lV`tg%5e(5rG_uV*welI z1li-Vkyx+0P`SU6xz)n>F=b+9<7Ql)IPFWqeWpKQJI35j+W$KX(3KjGf_t$A-BF`8 zh?VMqNn)6SLPg)w53T@O$>E*cADKj`TyfLg&xm+Pn7B+ zZhI-utqGLSX2)A_KWdXLcZF+2{Of71WBiX`1-1m9@?*|;)X1cdq&buliPF3BLOluE zA4nvVa4Jt!<9cJ}*EXgYi_=oPCeB`JB>6~qH}v&1c>${OPAX*cf|ct_kw4n=o*p2Y z>G;KF!B%N*Jg<+!sN5l`AHjXM*-{zO*Di4C3o$T#I|z-2jzE!T=>OwY3ruOQ`Ahpk zj1-sA+s+a{W1;-~De31KZ>s7SW2Nz(>~h^9we`pXu>u{Fw{NxAa^3`Q<7@8z!qytW z_-}2D)$X&zg2?Jx68&1A`Y={pEv=(W`vW~`$R*vPkFH&&Q+4{Shd~x8Y5>)FYRw8| z2FIc`O%L+=rM}e0dG-2=K}10G_cK5L)B_h{c+ctBaWiZ+=Na%%5*R5vS<6Ge!LV^% zm}KJfACbIPr%JE!V);%E9rj?SV=FnKPf0bVj;9Gy~S=I#@lM~>_I)U3vwrRf9y z;cTvlgCe$?olVxLM1330yiJ3#5jQ$wfQ%8T(E?JYg)Mh&@{#cZ5&x-A>(d zd$;er6&;(8u*iC5PKTp=n0}c|^%}UsroFh~pC??$r_sx+oSp=N#rN_tc(*#rL7Pz6 zL&b1Y<|(8~B+vV^)--AB9Hs3=l9fiey)DM4OOMX7{|`-H0To9Rtcybk8r&rWcXxt? z;O_43?k+zr!DVqsg1fuB1b26L7P-TH@9vp%7S7I0cUM(ceO1+6Jxi~mSbvDMR`cOI2TS*XYHXg)aS z^WJT~c-tY;aem8_*P^q=nq<7a?z*van`u}(e=i7-cb@(B>HJYEghdp1a)0ke2>E7A zTkza>WHR~+Z{JloWypGzKTGIh$OoI?ouRu($0wC(f~VEmzXNXi8>M%4s&9w?m*NEK~Zp z$9$ zm!mcBAimP6ygCZeCE}!!ayD?ECFIAOuf0C~r@-zsJ@#LKN|$q1%Z3Kz6luN)ndHQEJcB zLyou{6v3uF(N{E_$?2yRcFjAv`p!?5j(?7k^t4WUr_XEfW~TtDd)MN0CWRDb8?t8W zOC>XLdrf2`L-_$k=9~Ht^9Z^S?msjMoY8+-f^6f;_RpnJrNwmz-$*n8f==+0^-F+U zE(R~E0yE|^^Zu+CGB+CQFGGTgnbnn57re{>v6Woc+uG|Wd7;-&&HE$g2u?MAY$KAs z>e$MM5k%#GEe_XzTVoL6cB&fxHG@=3Wtzs%JNw5v_N?nBsx2DVgv~9@`|dt%i}NcS zPg_&p;G8+-N_O8EKWFdo>@5FV21wb=1lKV@4wXnz=`<%^eV$fI8pT>2LWf_d|AK^- z$Jp&c8uCHo^46Zd;%Q8r|I&Z05#?}JJ$tg#&)f(dL!5rxV5~-DFBYPB~$b<{1L)MKut6DnVWfdLF~$Fr|Dq^Xl@J%^E6Jz z@yt=5!kI`0J_bfUj&-{Q5;C!eczm+gV3WvOmPBFlzUi$oO-okJwtGKkv!2llFFb*N zhPPAaLc@3GFF(IvBcb8WJu5iM{RYA7vJ%@W{SV;@#PoXAQptWm4CVrlCHrqAYs zmEXGq!~ygBC;Sch(bEj+CMr3{RLb^-I%&6sGbfQxg3cpajO~ZFcAgzkOx($np!&cp zg@anLxBIrZ{=f7cKH@?8el#PJ%4{Dp5qhJv`aX`J-%7g=C-=tHcRW6xjxOGuap5mG zVEUQ&JqzsZ`3X17tmm+IeGU?9Z^`HC8eyZYA@H+&EL(A(*$QKlInMKAKeQ(Qtx|wX z@!tEEyPB6GqM4ip$|7JUq6uwo-tgsTuVWivUQuu+R z@C(GAQKwYgJH3Qp?u5CY-jrO`7L-rcSJYV)p^8o$9gXISWEw)VwRJA99Q(pIc?f3( z=-*5rG8KJG_nWo-5m69h5^^h)NHY^oL?66cd%#U7lvg4_BetQUWaE7LGZQD(qv>#$ zf&&@uqVmpRrypMKVjliTWV<5o;zjlN`p>}M0-_DhY)5O4Uzm;U)IuGq$MX%I$V0k4jW0>C1a=A zRNAuxW=(gIwpqRdI(i(O4}ZBb`JexQq;U$lFZ$-Yk$VGb z%}Vg%Q8${Yx+>q8hV(mTr;MQ1u#%3zIqc21|ISAd@#7DeNtybGXV>=32uAt&-(ypY z{;uWsA7s}xLd)n>8?}e$>epI`j~1CpS$>gxc!Qp6N)u%4pNV78(>XPEvWyIp^VvTg zbzEiSiP1IGYA^Q594@j<0cbcf77PEUCg|LQcICI|KP+;V;ADc;2gj{&hG5aKEw>TP z=I-sLD~6Mq5a&FDEc^e>+<%thY)Ii>Wh9IV!{YOKiiDZFgNsu0 zq+T&*HHw!73|>t=_7d5+1gNgUIv&f#!@oRSfCCPCS>wyp;pq~Mp62l$)B81>xe|Us z%-<>aq=}|bQ|DUtUQ}nii4N?s06r2ojcfF=j78JdGc`bWz1+T^nXRwlwbfs~GH$$Q zY0fr1G#~^68!;|Ys<&b#ppZx`oY$VmC^f$&f-ThOf4UX^AS}07=+b1i&ORmE2 zxylZR@KuZ;`gD z#a@nsIw8H@0PTus=bOr&)mO3H@eD-xo4C~1f8v#GWE~Y`kYkIdd--^XGaRz`AcEx3tlkuST);uTmhqEB0cg^CH+iN~? z{CxLjJ`uL;m5D{r;!hZ!g5%{9w-Am$r!9MOEo@ySUVt0xrGy5}KeLReb&d;1}y zvB#Y+Q^I36)zQh(5v63*12TI%+5VqI938_Vl%ouS2ITb$|uV|w0aZc+SvP| zi|;oH8Z=er@W!|~=j^6Rs|WhGli@G6+25Nepg$Al_tjSE}UG-L&S z=!J88Wi1$$bBzMMy*`nWRMTuciBCg+f!;=S}dpqnS###;WJC*z+Te5l?7PN+?Ll zDU`+POsHiI|M#Lnhp@sR{zJh3a>F6?4wesHD8S|KG8pg(ci!(=G!)>sAd59JAD6nh zZq>W4C5q+S?-EWx54P#J|IOX$&)xM77RSbrFr^-GE-htwJjB3Q2_s;VV=ju3`H!Jh zrsFD{86tN6A15v#;tqsqLIUIJm646BV2N=sEe=$p2-xpCZ?A!hNE zrP*Evc_cq@iTpch6yBf-^~mgEgPD5rF#L&(k6u4h2W~&}<~VBGKMAZ<#-9y*!_2?- z%a$xZo&YcsN`~ys5nEE*Q((BP>5t3{Y148mFQIz3lN6v6LyLA)neWdsNR2z9MuWLC;)gd+iSI~o&FwZXe)qoUn_j!Le0?e%Gg&phH{ zf|e1+B_Bm%K2h~w5t9~Mq&xIm zI-dJbz)svl@$jaq2mC+61ElF$yP7IYuvYqw2+0flZONSa@I^b$>4&k zV^V-=L)Tbps}M~y>~Q9Jr537}T4_yu^CsvmSN0owv)2X&=uu*~`uG&bn)`T;7vckBCFX+WcKNAQJ*EUMFhmvw5xJz~XGk{C6Nga%z zNXg-dYTt>o`Fmk)RI(qIGHGBj%Z-05XEmt+EB^qA#DvXV3V&9CUCt|(C>?;tGy}-B z+Vu6fI)}{eQW?3Od&8GUgC&4T>y?)FDO)nCa*Gn@F6A+oi*(*ET(SAa=Pq?(AUzy_ zR6=qAJQ0=$t!1B-TCQzdg83_D5s9uLo$IVi+7~~DWb3?@?PMc4NFJ1KFdu^Xu6JyD z0%$q{KwhZF9GlLAC;%bmz!FwZgCCAuI?_|UM zI5`6wBtDVoU-{PUOnaYQb>5%oJl`I|)%x81lYXqM-sqzfd(Q!S=>Dn}KB(egkx~2Va>gqu}ss+Jb(ty3S??S6+ZlGj_fJ z-9qooTvhW};h*^*c{E6TyWp?Ugo%`+3*hCIsGOp@dH5H~rCM{;gB)LcXE@uF3YP!$ zD04VVV?9e{VM`;@3Rxpx2t^|s5rJ*zUfUeO2OBJpHxJJfbLH~>LQxy*PY@WC#Qd~X z+r4`}2MtV@3m0=F^UgoEBNtLVxgYvc#9c@4abLEkApq!f>BSD>p&KsxoG9DfK9|YR zaSul0($CoJ@zhN!za$Z2P8Seovu}}c3q{LkODPL^Q15Nfg>yxR2f>POAvLdaKXtQ1 z#`#o4HZZIz_DH|`W%u;-KnuzRk>`&V*#;jsiMy`Kn~sT6KsIYq7WJ}INnQc_bPnT{ z?Lxx+;QF5g1QIK5LP4fCXC$lrS5#w`4=kT?I=Ztv-+0~q3+}5x7waW2{C8n-GhZ?R zV8S&w3E{5!k4{=(_1z`lxzwmHcYo05=$;-))OlgytFDqvqYc-~AS$N+V=;Xau;c^U zk~XqMPf-o!jwRei2s3PE_X%SccE&^Yi5`pSt1!CbKZm+}UNgm{OGT^)MIu!bSfnu2 zupKIH9S5fk-Xk25h>FY0QMgJb<<*Dsj$}BW(F>=im+|*;nv$Hk;|?7CM=I3!JF3;U zy^7z*x+}R&??(`o9T(lq00kv0c&H@17JQDCNmN(g>p=dU2aHAdTw`WyoJo9kF|2kP z3ns@p55~ckJ6M1Zqy3%&^4%6x8Ig zPW7Kq0gW46y>wYpty4ni1=K0HE4Bomv=bQj?FhYtz9{#1n6y3ZzM);ntmLn zDEDD!^@ZMkef8B>Tb|zlJs9{5rx7(8H5E1g4liZ|p<%*o?>-e#|2fq9f{ETPWqdV% zY!W{L2P^f#X|c|S1#{$-a>bV|0$--yO{(|I-mKYme6HdhL()WM@-8eKYIx!ffXoXE^1-@V8n9Y7Xh!{dXR$e4Y2Lk>mT zHo%X{_fs-c&^^hW!ZfwK_F=gW#_Q4h6cq7Q0uVuUYsPi_LUN<{5$fw2WXfnaZcpB< zxi=>lKm?Wdb>O4kW@ZbxS?%5>IFn*KGwMf4wo~YiTlRPw&p4ZI5=j4Yrep3~`FR=F zjaSI&k|hT!+Xa9-+{N6OLeM0#-J-{o+UyN$aH}5lqe+gV`Q5=t7NB z0$jOM4C`7Ljiu{5JNA>0kFP?Qc>zX!2H*O%L(zOk4w&kAp5E_nsbjM$kRmLQrHEb} zrMiLmmPG_jj!*CC3)&-P62QTgW;Q+VTIqei@H}{$sp!qMEF2P9GPDo*xOVLCq$^$*8 z{~5Rldy504XDo(P&oOjbA=ymTKhInu*CqR+?gP$21Mv!UpGp#qj&b|jB~oo%*0`p2 zXG!Yv<1}= zFdfUJW8plX$`=nc9ap|;QEiynjrva2mnCUw*Fi6=cpg~GUe<0dtS7&qzvOfAHN3Us z+eeQI)Wwm@-e3f2f_1UrIPTD5cnWP0Jjq5c(C&+=?+fwxxJ4JK0zId>Bo)MNH4Q=S z550U#+xHsF6p8Hr{g6?sxUY3c7iRzM5D*}*m}afOI%KBvU#2eW&LS~h2bu``r^n4d zYz%%^>Y4U>bQot1W9h&4&)=Os<|*KfKN_@$TTjY5bhoLr9<5&Zo#w#F(wr@0yn^2P zce6a9FhXMQ zfa2I`J zkiR}5j~vvyV<}?qP0Q2eg4C$RTi4mYY!zJmp^4yl`zNYNS`Q#<&01szuQ+>!WFd!N z9OVkC<~lp9TH+Qop~!kPrZAQf(~;m;F?g|iIO%-LD2)MmT@iH207L2?$lxGCT+SiG zRfjY`BpLkK*{|3lx8j-ZJ>#7UZ4M<_L!4gM$V?n}a3Ddtz>O6jZ`t7PcKK&d<7W8Y z0Jd0p$0Hg^I4o|PWkaV0W5|*FwACjNG7{Z+%W39pr^V&?36REeSz-hX<8U#nM&nZX zUG-n3X*AWoZIsygkk|N~1(mtd0t(hS!B3T5&wnG)*4 zpScTJ(Jt|J&F}zJi>JH|CuJrdmfKk(!4PsdPW;vwzOb>d`oO?5y{2n>843!OEW+x4 zLecNw?;bPOX4KSdnBSg|fc~XkUsgkeV$tEP8*ORadhXkHPDNZyRKR0e{IUX6wbiAE zB(V>X<;yK2{Bt6MpO5n2ui%BCG6)mTnGk-O9=*)?g#q*787Z=RDhAKcLpZ_FZgZYp z_}TCIM{W{-i^mefXrmh4pFG}dxmx>Jv9@xB6?4pX(Z;L4uAP37495|1=~FE!@dQIB zbqZPoQ2G^2v3N0O65i!VSQmD&jaId$(uCf~6? zEsyhHTs8@`=6bOK%2C1%gAofsBWkq8u~~!zod)P)MYh0+({IIBua(8ogD6jBSt;Vl zew~^5{T$UMYsoQ*M3^4mw&ZH1$Y_Wt^c72_y!%u|B;Hk`RfnF#4`(K8a~@9`f)*q9 z4AtgUxuW=2NV|P$Tj&D8>FR}j_}BiDyARB3#>s@d!})(lcIn^6j2!OOH`gzSuS&j% z+_E-6li0eftk`XwH8hsTHHFGCK==5`rc6?vN`ARELGZMN%6TKY#LoXabd5*mg}0(8 zrsU1HW5O?{q<4%FsUcNw^_Aa{eJq}{N95L;q#~H)b$IvpN>T0sMkToE-BM`dRp(a_ z;m2gwcEgJ%NZCrE+leOjGNHmf@SEEV8)YvDAHmWgd;U9c-;miL>S;Q9-ymE`IavA0 zQ#qupZiQ^x+8F;jovbf`LKvOwIlh>)$1=%cS>AVsG{a))hsWXf%<6BeH&;9iB`AJ<{ZnJqL58J+NuF~+^5w>FtqB8+&*A^g4VSHfC= z3&h75brxUwtG=_fh%F<0o>uqW_l8pkojFWov7$z5R(aMonT!+swLJ zL_zXLy9ao|hrnS0|L=@|jAeupdLy?{&YrvgVc}S_M^XmckkG@uZAtEv(MYa~RE%p` zJ{h|w+cahK;-_U*macJf&l_me2mYgjzG1u$C-Jt?9Q*TwtqS|V63~JxdVb?_XS=Ph z7`l9&ojt+5DS~HFdpTNrj0*@oAN1LqEutM?tWZ4;W z`3ZxJ@btGju*Ezr0M*autIi!!RJi|gqx>R)5!_Zlu#)YDkXOX`cD%eDo!bdx4Y=7?59V@wI554f&z{HBnartXtx zA<(1Om=9W*1LKEBX^**lM7KLjtnT$bHANu1cf58P=cC&!@vnZIaP?(lR?;hKdOI-m zA;x9<=l1@qNeCW?7dH&tRBK+M0ZfnyiC4hci^kopI6#GOVf46q1@m`nQijQR)1#VeS=7Z8h}qjAi5MXY&vBdHsBJ!*Bx~L5-Ax~f8s;0 zYw6@JlHh1D{iCT4rZ_tJV@ibB`!n{2zFS^)^Kw}%Exa71L;S8IKXi9s|Hu(|%(OFY z&PLlI{X(j5KE9>kqeb$&PK9vy6_Dy$aK!+9Bk8`bN9k-6kEBAJUc0xoZ?(0bmx#J= z5PZ;~98EGY$f;|@5^;%^(GEd_$GYT(;YSHA4BWqa1UH~p&^}bzwx&|)+XA!XyI?f# zDYIoXFiW<7dT*{m!?$XI5P#vwCzMR0bRk)RVgh&WZ`BY1nw)7Ljl!<%#c|?Uz1* zUIkZgh+676|3iD#1XKs*9kSfXeS=vRV3u~1KUg-AaLxvR7$swZEsGJ-49le-ix;XU z(gcQNEq%YaX#Cz?#|x)FTnQX$>bK{tIRB2b-itY(^EQ|R&@VxwJIIvEQ2SL#_jNY_ z7mI<^lrP|fwCWpWEnn6F711jgo-47uf=Jd928bqFyBvk(`B`+WK7nBo0<3ledJZp2 z@vr;ah*0j9Y_MuSSEXVWd%1nZZ~ex(T_EBQSYNsjI zt_@MLmXQJ5D!)waOBaB=TGk?hL!VA(*XKVJ(tJ4D_E2JO1X}*&Vq=U0``Nd|dj%5s zSru;lnjp$29+YqW841B3;6QNQR+U!F-+jq8(wd2x=c2D;DS5zc^9aaNUABQ-@EJAZ zw`HAgC55i!hB-zwkq2rvW{-NOs#XNjB4`Ovm_(EUghuj43UYptzid zqE0EES{WV}6lF|Uf~eQ&9>R1v@q5v}kYDb91F{wTN<_L{h7sk2jEm7{ z6%0Z1{q-R&zG}u{l6s`=4=EP2z73+vU47%MS2ad4!@-i`^svOrTMsl&4`%5|B&9rN zFfzE`G%52Rns>;=`7;ciVf>zPfb3p{J!6p~;=Y_D$x9&oWr2|AVuOJAz%>8WpZ=nf z5ix~ZMn1Y#wFbKe_<#Qh#Eddbq>?Ph?k{=Iud&(04O3)GyT}f!<20&`0R5c-0CN!BIX8k3S#bsPdQpaYNXXeb$&yH;UH4!8oCv3QJ{l-q|lTf83ouH!+9st{$5 z)FAt{y`2Mn-nLC){o5Em`UZP>mZC-9N2CjQX;PV)$^%~1ZK&b7vA`JKbMao=(D%Y6fiX^*f{8(C_EmL*0B!5vv$!eSqXpnSM z*(4k*w3!q7v%%tbTvezYiSK@HlO1r=-fQoeZEcGN=Apd-HsiaHa~@;!`gfUDH&sD` zZwFvsXBt^^pK}T9K8)y*a$8sZ5w)`Aq``Bq{w|*+JPg85@e!g8{#^xsu<`SY2`+zH zD_J5pN$Wrn%T5_%)G0mEyL(2Ncmpobpro`Oz(y%kLUD5AFYEQ7@cD2_bV#**sD z6X440VdTWZ*WM}G+WLT$gRCPb5~4WnuWVimu88ux=Dy@Oc>iVp&yjw|E7)Y)NRt(O zn>*~0*>`#)1!VXC=pt`q&E#B~9E7=$RS55vKYNJnWQa*6N%7=bpkH2Q()bk1cI|9F zSU{!3SpL)nczB2ixT?UMD=P@Zs4PN&L#h$|!hLjN^|i`|qih{1bHqSBR9H5cGJAd( zh7c|^p9nGJ)Lw!;bHpTI(0}e7>T9ZBZ{ZM*@z16_c8Ei6+a`MLjp}{YkJ_G zcsNDjHr;CQVYfEX@tY3aVj$y)ymQ;W{ISmKC=UpiK)-B2@^}L2&)TY3ycF#a|Psjk1DDZtb6K45U)t~Jy$fTL}(6rCadmbl< z|6!{9`|b*{LS}Ta{z*UD(rU$7klNpN+xOLhIP-2=9SPW8{6d`!cgtSQrUQVO`7u`Km!!ISF z^>WqFk;>sT%S11@lwr0XpFzx|wnT1_c#k&f0U7*+|iUx!Wg~wt*vKrhw9a&fYlP?Thdj-_2BP%Xlw@sYK z{vA!8-0iFLqmfAA>I}K`(Tg+p%YWSyQj40?ZVq2lm?H+X;khsk(l_hx-&lxzQU;Xp z_`_sVwiM9*dGw$VEAOyy=qZ!Tv!mvmc3!bx6zYXp-C?M|aNNfQqUtU)r1)IF%WEDf3VtrZ{f=JwR>G2I zl#b|Z&)k3F$fmT6SiEcME^jsx2t_cRMzHv^cp243gVIAbbkOCV*d9Nw`ay$@BW19A zXF+x_W6&Y4wcynQsBz}#5)S|ofC4PP2FxveT;349Ga<1CNwXEkfiH5?MiPZ;q)QJK z=dj30T_-f@I-#piumSoF_5DFT~%-8uf2z+6n`@3@e&Ke8G>e{kN8FNTxaw?s1q^>%fL-E ztD{Dx`bBBH?*KDc-Sxt}dPlL)7v5iv4S60C>S9eei2OC}EHD{|hI2glpP%Zb^N7Ln zP93*L9poMcCiJ}3n9Bi(i6~?wf%$M}e}z?VCrqL{KMOzB$AD%(J)o*ZuvRT9&e~s{9;e zZJSf>`H*oBeH=r{|2O?wg$wBrV)t()iT1T;|9m?0J;!)yBbi3Y&?7sRF(w^jAXg_; z$tz+cXH#4y0ZgSo7GoQ-NOHD+M>Yo5h2CfN)m0w_IbR*NyQ>OMfpDuGpO{X)JS>~6 zO5CO&6&9b!`Z^FrX(sU0`%j{7fFKT2+*^K$5*X#Y=AD%+d+XFuRY2gQfa|Nql8OhI zQoI{G?cIB5DNhj}AFeB-c5}xW6OW1TLE&cAc=#@IE6T9vCL0vTjotR<+P^B)6OVO% z4?W+@vvIQ2Jbu_{*cEm^7+>2?06&ZJDe_kNh}0(glc-u(G(3U5>w9h3gN4)R=AJ#x zNHJ|Br-cm$t@N72Qc5O@Y2m{#xgy$Rs0k;G71o>A8u?T26*cXEMV%5iF|3xc$!Zbb zhypI&8m%V9PVi53nB%U6sb>yuY>Ttv93E_TBn4`C5v-QpBa2})8v#9*edHKU!&!cD za)OsTq7^+Le3@hk{dZ{zZFjXYWN2I|i)CdiKz)n+69g`mE4H$H9^q~0UsFOnUz3t+)jvpyd}So-;@nt1ik@h16Fi_n+y$XdTL&j;Q`*3T6!)jUHp%xEb(e@m%t z*Y!*B&ECrQoN^BQ$`I6gLk`%ZD#FELy<3@pl}4R*)iNalpfOj}NAwG*Fr!|mMRs)} zB^Nb-YQf!_EN~&)r)3XUkE3Mg-cWGZX=9)*hu40zOQI*1zAp~%NtgHVl&!REm}cEw z3IavJN*ZDGwO;`Edb90`*b7%1vn#dNwIpPdV{I#lj;((%k*ub-`^Iy`Nd44RE`EJ} zfZ0mPZhs-!RcOvW*mnBn+y5<+!@Q-l)3>%2bkhlY7mR7|2W$2oTHFws;uC5x=yvg{Pr#FUSlJj30Rs1LR9~rt5*BU z-9O-&p%|rY<_B@AjdS+cX|1MP`e=ee|CmjwaJIg#fb`j34bx_N4Ez+Y#{cm zTS4Y%rOMcw{t)A=N?efm0^NG6nq#-WC5p;QIZ(=DLQZze3_{keS=)GT5r=!T0U#Af zGt5lGF37O?6Jhxx(be^QD;cq$WCF<6@r5FDG6(rF`9$5BU{|LuOMf(UX$-Tu$qhSvv!2M>4Ija!Kf{gcB+4icRTYM~Sq(+t2%@HeL( z&n=?}u1S=k2^%R*jZ=qjGOG@qG-vjt*Kvz9yWgF-0B56aU8x(+{SfCZ$NQC($4(`+ zwJBP9&j4tAmx5t=Ypd{P4#LKXkkzwDVUkwoqHR}vXAtk^I7@9qFM9nCbnYsTa_8urGH{L1E6-)TDKL1cWA61>*@$^rg$m+`bmr!It z(Ha9f_-SL>CTEXWp>>eXP1fn+vug9wmm>SRio5E{Ack`i2R5ce&2ueIx;^55)BMz@ zeQC5mtbY~Yw*A>V>&E9gR1)w3bz5+KE^;s)g$$!zJ%}Ztpsq3O$eiJHlDx6~iqjn_ zZLq`NpBv}>27v{{6pERD+J-2N(LP;5-);53MUy3NgL>^if`w#4nvi3u=l&7< ze#meR)VdcrolFW)_Ag&z(NyF_c9WSePZd-cV&9IqS9c+Wu* zS%yD-m5SMOtB5k+qehluF%dsGgY!L|fVYMhy%M~`g0HlBTAW(v-T$vFR^VYC15;bx z0xb9y+F1YW@-hw=Ac*riZnU>7+c}2=zXFFyt#PE$?0;3aY-poV;cI(uH!E-=BXU{iKcS7=cQE-! znGLdwlS|Y~IRn^l+OathsX5SV0Q>g#Cosqe;@mYkW>GZ=#N#N#h3cw3u|;mw`)Jjy z8&R!fmPPe(2Mtd$fW?I?!%c^!8hL!ei2v&R1pTwY&KsWS0hJwl#+g=yE|*$^r=36Z zRHvhGEoGnDt@e5yh8PZL2juTgbxvv)~Fsv3Y>t3s&H0^ z?Uv1RLapKy#6_BGnHVG+pywP{wt%MnL8tek#}~(gPRZ|Y0mcC~;D2=Wjf@{ zllhS^1@(;l97^bjBC{2`tR@cldciW1vzftDIq-hAMKJvYTUOvnZp}yt85+e_Cr?Ut z%^=iv=1;jDy*x&lqHSy=8{ursSlA!KHzNFYeFJeCkJrDEF;vpsALYA&E{Yvc)EYi; z53Mh``JxsX4oUT=XrfJ8`s0n9*C9nv^j7V`)IU@+pwikl z3skSL8MJF>njHK~Y2?BbT{94$e(n|e=`QD{tsUb>;54#|`p5Tu_gD8&XodyZFM00A zxBu-Y0HPJ?QL)!cHSR6v{@uIzx8652jc=@9yA961c$2SWEARcfxPgErz-j6{7g}&? zwD<2&sZ%Tn{%!nr_Jeri9Z4lNkcw-wsd_Haxp_&f5ESnOi@2;C%~!6G4p%7U8xi)t+PapBv}xin>e2 zFy&Xv_J=)1ww<+%TlE)J_AbwEwNFws(>J(hq-M<{DGaHQZe}^EZgr zL?8iObj%Br+;v*~{$K)7=WlA)Hk8G_YvAc}rV*kmb3H|Sb%igJn~V}g%`!+nU9x|1!P~9l8xU$G zJ*zJrA0_{S5`V9W8#|EsGeMW&TPtH^_4_m0YJ;r#3!xw{PQ!#-Fn$c(z%OX%g|2A- zLu^}1iMA3=f7&tu(du&-?sskGGZxvxpVfz6?%wi9l<&&Q&%CF2J7$IdkV2jPO7U)r zwO4On<%5$|bR3;e95p4mcRD#P^h!KVr0v#F>&X2HvfkL-k=M5(FuN=MW41`9RGEF- z5lKD|S6otQq&^93Sp?{In58i#Ve5x|l^YjrYG}#|VKP&NO*0?a!e zvSVk_!WTV4(Je_c1c~k4dG<}o`u2wR$M@M?TXh?a%?bbe@(J{H=vflS1DP0CK@nBE zcoVSW<%K%W!sTdE3vcNlebi;OHI2QJ*+T#&~t14a(EWCVE z2w;)tr}E<(0fn?qvv*VFE5>rSB>a8rQyTT?cXPrSM}jGng?{R#76NvbpG_@ z+(8%(fnY%4LJu>hS{D?o+j6(uGIcqgf?jVIk*cj;gq?{uS;NZCBe;a> z$ZVNwN_(dm3<=QsHQy;4crJKB1xjhNGQ5!$mYK>cuwzZk-u44s!;-XAY@_Qn|Lo; zh7V0pwW>-CQ5T#QTH8JSsB&TnzoxV@mHJC^{W+{^pn>8wZr)L2yJbktpb!N z^0z%f$p$!E$wDoDHDPHNY0?TE4N`%+O_TU~@Bp12smj;_eJ*rKfZYI)(pSxy14E#b zrSS`|dbEgj2oVs}qUiL9@j`;ASl2a{r)Wxo@_gPO$Lq#s9rxG(T_o2y^ked@m<{Vf;2E zf&tNkr^SkhK-(2H==}h3CQ5DSF9JKm=_@7AhinK(4ZxS8rx%QX<|WLJ9y9`9LVAc0 z_lSy?2E*|B@Cf#Ziu^N&|4ln|T&x&bK_Q&5{zLd_Q-q%kH82PoUMz!4(FAq0hed`q z@+S5Y;^Qn6v+CSQ$}y(-;w}zu97z)LcXD?`ix! z4C1SY2=<<6?%O?&-@~bgMZk;Cl`}!G)KIW* za<(|s2oRAO{7>s2o{_KBb?v3u6SFVTWR*H~khrjtUXn(Y^M-r=0pPHt{s+HCXQozl z!{H<#991|OZ2a?42{48a5%9itsp)cl57eStR|K*f-T=}2683Q;fqST2ESwp>cQksD z0pN!fZ!|^&U+v)i2uZqr@#Dw|$3j#3&#nj?nh+z#iB~@hJ`~Vefn!c#*@V8BT^49q z=kG%Tu#ei<)CM9;X5}mgrEmBU8~maFW7+AFgh_hbu><5*G>&9&;(h`|vC!V(mo(RE z>=Z~FcuF6u_agadHc;bBfK1s9)qh7CaQdt`<$OApGTR}e zwaS?=Z=5xt-eA3h=SN69MGHG^iS8{g-SVd2I0xRhrujrDiI5Wy=;q><1?- zXex^#r}}(V9TWd=Ex_L|*~^T2Sn#*&XCItAPf%a`ULzQqed;j~{sZH0#Fp$5jBz%& z14BR_*hm}`7RW~@DKXpkmlr$qpT%(_hXF5&c&qjQ$w8n+;%N(RLbf97@7QdEWywrn zb}hLoLMb+HQvwPbjCoek?O=1p4xTZz;4S+(Qu~g=hbVZvb<9kX?>#w2^w3$qd5}q|nxmE83)4$e*N6wQE zadbGEUx#ZjaK^z;9)~Q~w$A2XvYT%@4t!!GA_(=8I4}B!aSqp~*tWA9MT);K3=n6v z4Hk~KqRiq&MnpWW>I6z=x+k2BPPdk$T8GQ?u&&{GH9ja#Clxm6wiCpOBZ|k0&$HV4 zvR#QKR>TOn*vlylA=4X~SDSb{yOf<$ETtIr%&S=~d&N$Dii-G2^V!sCOrw@)CAG*` z6@GYrHy8iw{7oxyW94zzy=i*28DE~vIYM@IM8vO}j0R##8|!aNDfz~xR7>ltxq4qi zvw#!aVRQ++Bd8K(bQh4m!caT8%<#JpQd3>97|(HPQU0qLRZBXT*$IQ5aEqG z&=gtXXXCuc&;R@I6I27dxyr*naz}4tY@P=BR^z9Q{Fi86T&{p|oS`EAE~V+UkKtoR zL_`ds&s7y9{iGdiph0%2pxfjQ``*_N(>3a`TMHXqA8mb9=;+NC3zcA;B9dU7X3QYF zvRt|`C$6rF*S^Q%WQe1DiD2%e;`X4lp8Sub6reVi+XJp-f3J&LYEoqAFud-nPELkN z*zrV&)kkZJ1^ItdcUr(dRr(Jr=)Yp17(t_S%jDFEB_2=c$r1^{QDfpk2QBg6wYC7w zxE9Kbi;0CGl@85QaYmO4lT7ctaYw)qT!C_<@4U<{JZggjTdX?!{%F$ z-no8fhae`5@IgUq?uU1gx>0!o%60@SK8CA#u0!#do{FU~4jhp-Xedp8O)`WdT2V3x zj?B%T3oQ|QCuRzuxfE(lbaeCt;~r-&+!~hAqK%q=HpESKJDOo7LQqWl=G$wW(d*E_sJ(b{3f^Ij!y+k=WAWqhxInTwTrY?gmWBKDc^s1R6JLy(jAUa?eRE zQ2@mWsYA(ukJG4NcXn76aylK3sW4!@nwu1nHt4H9Q@1Iu)=-ftZFL2x(XK&TE)6f4Y?<>Qi`o4b&0TqT& zkQzZikr)u^9wjVN0qK&I4(Ve9U?bviUO~^EK{2g6j@lWcksAb3piEgq8KL zFB(@1mpv43XhSIGLrJ=HvMwr2FPg)zmE+KNyV_1&P1l5ytNmMiY?15QI1F=gXc}_EqY;&*AGe3^+jKFgIK?Xa-M+d^InRX4AoQUG>z-q_c6bY}()X$%= zdl`4scTlToRLr88pv_T`+xk9k=&wz4Du+hz@PYXm+U6_D(k!=Gr7aHR1O;B2cl(eqp#!0Ez10oAZKs9PEzz6wDrBgAA_*`H3Vq9&$ieLX111 zL$W-2^TFG?Md?(yF~IMx{A*+-75-Flg($gWPx*Z!rF95mP+~|DU2)DMes&8PsvQn} zB0$3Dnqww-*EWY**u3w2{AB*Ox8A|{S)3M{Y+HNy&9ua1d`WHM=8VS)eAAsw!ss_V zYab`Yw}h9`mL?ln3g5i1!*W_>iZzC~vNDV-QPa?Hzx~%?r{3V@@{fC=zc;v98A>7OEPnwNW-Rf%7e5{HYKUk+gIXRgNpgPmMH}?^-I>)A0 zxk`-5Ef&*;T)<HDK&U_*ciArw3x5n%(IvrP}#y$_3bW_Tq99WRrK(J~pN9-z9!@Cf3WTiFK3kUmQLt$DVr(Qi z*45XZyo{S-%0pi5_JChfy=qXt4*f$hd3eyAUF7G&>qgrQsIe$Y&Mv9rX{7qb#55qD z+Pg(>V~zw$4T1oFZ1U>UpKoH-PxhbPnb*EE=MY(Uw(h--FLXwFwCnp1L6PJa6aoYf z49U={g8i^=!F6Xumk<{FfHB>uHn#{=Ds+xdp0%FxWnw@%&6ICuHQrY9ZC9@Wsa}rH zSFmLQWkhn6KLG%1Ii@1HJbZlnAU9@{HbcjApWU4mQ0&U31IXO$VtbcTd_6^d;cBm}FUN`wmM#<&@)_ zrcWPaL^htit}KIPEfyUm*JIKk1r6mN_e|0o#`!{z(x@u|-^?^LUY}GvN#=GXkhO|K62tLDKg7M{@it07pU96E`w-LY zL|0~=X|&Qmze`xD%7w9LI6znIQuJs(8o!17dKu&?)wE|cP1R^ zrW4F)tz)a})D35QIGR;8<{)4v!|0t3q2mkZMztlbKn7)xIf010B*o`RY^>d>dYP)F z9}$38DS!OkvDKuD;p@+2PhR1&WtHtD0PY1RxM(G8beX<3zq5FAcONU0(~~@B0~cui zT54^LWkAOCQ8rdToO$K^(Sq%b!!28U4u2QCq-cULsKvVcDQAsd(BVKn4))@;m;`ku0N0l;LINb;TlgNuPm*L3f%p^Rxa_e59T>ReVcP^if?V1 z;2C}JMN1epydtRQ+03Z(YY!21nioO^-rB=N6@Nk|FuJdUvL)K}PWx8-6pqi)dOhBR zuTH257ataQ)3;p^w!;c$Z#!5c9da)G+>b?jeU<}{`SQhB^P7d9z=IS-6&(4MPqwP? z|2~Yp`_3V)L^}~>eHVIv5gXqZVI6Z%yvCGV1lx=-55~*u35LNVy9d0s0{z^f`THE( z`pTTFCN^GD_s8wq+#tM?T?zbL8aIJ7>rbMUBA38XBAY&6zm_Zm?Vc3ANlE*=uS39c zekU>hQ}=vAU&Iq>2cB)JjQ92aaS7I!W_tujeEFOo&E<5(5*}rFiAeSOQ1VJ;932?_ z^F$gNiHYCLn=d&{r1Q0ZeB{0|HKieD^4SbhlR&#F7k~85Zfa5ZSc^cNes_f~ra||s zL+qe(%-rly+cU}SJ@0ODZ&xfK^7ylJopkD2b{=O>-{2ESdYtyZ+K80_$cJ1_f~*b4 zbG?sC?<`2eS#KgGAL@v5t{oqcPGgQ*cy2UYJsh4R4AZ)wiT?wE)aM+dF^jb4QV{R8~=^-Blysk?%%hRxIr5QW*f)S7Kk}?nF4+H zGZtjVo}NXO6;tHtsAsURkN}QxJ&SGQ)>X1S2D@QDciWhIfOP>^tBj~C6o^h_G%mMz zf76OLJG%}ClAJSfk+0Wcr{R$(-hYkLMffEUWf-J`Rd>IbKF}BbKBYaxy^-eN@uNLR z8Vld)hQ_;IWl)00yaWF`rG#(M*GmD``DZ+P{)+3bjkDi-{!b3~=W^oRf6}Qxzr6ZL z`}f}e*Kh7gO`?)ViT`GC?CBtM^~~}(lS6#V4%&Jjv2qfwaaHaO_?^mpMGF~wHxa;a z1OLb=vWMAze>I#gX7$7EA|ixQVU>^?iO6U7SJHFGn7J6fG5qD7CR*X+CHfsAX;d= z#b`?VLEckN3c@Mx>(Yl0L##}?ZmPqEZ)^%4h{8{+u}QW$7G>CARxXBwk^FzMNd<+j zL@j#eb{VH(50I*Zqrl6Cfsq5A*^~0{!r0SkUfY0PuwraotMV)>DeXGZ;)fOi$2wO!ftrG@b@wafyCa zGfqXH)ym=yc03xxa-m7?Uk(fIT3lk3pp^QmyV`RBpDuA_YC03yRys#4^*gZSaFL7J z6M?34)hW-eE$S}n^-vOzDH%c#T!>ZBPKvRtzIX57DM%`AKZR_x;`ed#sv70xskVU) zSLl4M{xL}5NRTCSPfE!`NLkOF!0~$z!~d4m%waM>ijn1l{jDj4&59mH2KU ziYMXcFuujtr)?2#^9iUEY-avw9J7w0c?I*Wj~M^)dqxda{AX4>Nb7Y6 zJPqUoQqtJ*K-%Uw!|UE0VbXxVr~2@l@5bw7q^sAhbwKy=y@IQXTo@9N?Rfw_(U5O& ztJ)H!Qq#a?v!&2jd7ic5xM8(vw{I$5pdW@QcYRe&GI-h$Kv`~h4IqdUo`FHTye3=3 zK&F^(RMQVbQ|W~>muXymY+Q4Bs>SA`GC1Tu2R5XqC^U_{r-j{ z#%JpJ1JG=3$L*ymHy6n8h&A$W<;5hxxF3Er(3391_{7{KVUNjwL4A(9T zt7UY$h+`0){GCcvZfyEH6^jx;v@|RCtNYMR>aRn)q9}acyX(T^OIE2APZf{IMKeoU ze_k{j9@W68?W&%1YA8~M%KeM3jH5&n)&`rG{jKP|o)r+9o=Nh)Zc!aArodHD8m9$( zpnreXp>T>Y;Yuzqh|1sMV5Q=I4_C5QD%-2rkRFjt=??Ulx;FKz(}af_3g!aie`+&- zkbpyj5>0=KReb0`%6WhSe=1>VM0TE52p5;(5UKJi(aVQgA~=ar zI{#A|Thkj%eVJJGlqohbMh;RP*!^ovLzy+ssjYxqV+2Ti2; zUwzqZVWJ%q3_Hfa!Pyl5lH!1%Q7e5jSg{dJ+odRc#s=(WhERMNBY;_o5KaRL%cP`#+)a zDXW=vIbT6(Wx6>;+C=N_=R5~(u^L#T(*`_?UX_LaJZovZ7{ojnY=56}ga3wO#3zTE ze%&SyUA&~cmXMv{6@7Zf!E=&+uc;A1Dx(HcqlWx|<bhuR{7$+(8P@CC58LuOedPZHL|*k>SfTge{!E+phQrE zU8YEoIYgO@<*or8C-rrLWVu0((a&rdJv4*MsV(_w&v-Nr#El3>=;Dj6bcg7Wg`RQg z!N*+L9#YCR;}JUN?~Q&JhneT)l>EH0xrV6^@DjL|lh|r6OVJFX-aN=gkUx_BH7PLN zO7={aLgvFo%dWe?9Q-hhn%3ZIJ_EI$Ct-lG*m?w%jE6qR`~#-?-drVW{U#$KYgYC> z8H=8rq2bR8bRobM{|vB56Gw>{$GMWy))_j6iOOop9m!X0msQ>z_3OY34Nq*jZZUJu&@2S=hwjaeMWM}GlwmhT#Q%*G3!cc&Mj#h zqcJ5lW+fc__PgEd>+G*${$@oz!)*+rgIR&}E`@3O`oh%Hd<`h`h05WPG3kG?Ltno3XuFxg>*PC5BbQkGg$b5Gx#GlQCp!F4Ar2hBW za?@aflT5KSKl16|fS6fo*ZHVOipPI*KCp9}U6vE)(^_7kHNWbgxA72Oq5Wo;&H$8y{oA*{`9th=pl*I`Y45=9`8?edO7W7K1G_D@ zd9N7surtG;xMH=}FnYIs4q*Va@RA;@T?H*Ec`g!Xh=FMrwv#4JBr{lg%B!+WXN)?dFcOWtn|G zBtdedfRDZV-@-Fcde(e@n3r}u!|5JdoWp0VsBPs9KMk}hbXMUi01|4jd&Dp^NRUU_sgswd-7*y}%) ze_-U^{reaIu|MYil?hN|{`gl3K&|lkKefXDzi*ZrA?;!)Nm&{GP!qA}`;X+{M9a^z zLjh+mp8XQB7&A*{-l5*1rzF0-cX?@h1d@0Nh)(7hO2BL6g6oiK=J4ZTccH+o=z~?$ ztgTDb3oHQ=EPh<(veT4MF7uQ^k-%UQJ6laR9v#`!rrX?C<@N!Qb|3KBgai^pM5s-~ z+nPUm_%(lJ0z73EDf{-HN&L?%F!z@y%R6_b8`nZsZm(7YRwR%08cK3{cK3FP%!m~S zyP!0=IeGY8Nt;H-Jr_-E~l6U-F7(1+AjG^;HL57D}YMHVgyMAE{em&~`FRGIniW zP!Z*bKO!(bb{}?(FFbwb&(Bd+_4E@9sW#db+fWgiiC!QosfE!f5t~T?Fgh zXi|=W0ZKJ#n%tH-vt7hv*~9n3U@w^yGLD6E8sMW6o=|14tX=%xdE^d)MPfZ)c$ zbA`v6BJfY;S5C6Y5VHr^yQB9vK#Y4?tr*7cl{6s040NUmV(b+z)O@7YJS0 z`tD3q(?j}P_cCX{*bC}Lv>d=K{J7;+?lo;fpt#;&6a)5@>vbfa%Qak=`;St~f5tE1 z82_kU(I*>wM91bW6n-siLFPyrGN6X%erb`LO~LtA{M8K z_F^fbD~7*SnJ4O``is!$i7Xir)M>0h^dyPb<@0GmHVRzZfElv^`(tspuQOcSx34Rj z_gCM#Sf4b_SKY#RWnAVi|RpPRr>MZ8qK@ z(`TD}uVuYjF}t|1cYzcF9M~HANp^kfl>4vZJ{k`uVN< zi1UTPtHvj9W3v4#;2&FINl1!Wtg>G z7N9o>dTVctGlkrW#h0P72Av~hQE#W3+%0xRckTuQbySCZa-A}27oL6x<#5gEXeJN11f8ZDr!UmP5YuX!eo+6zM!OgD^T>04vKBVJOJ2JS$EyUT})lJ>Z1KCHZC zRwNkj;w5fnnswj!+Q(;WrnG3PF|zP*!>$=5%k+H;bTRR!OqFL?1a~1n|9zzfw~kFwD8Q#2T=$_&pRi`CPLz zUwuLE-3@L@r`>K6koW3etV{_eGsv7_zzrq&-;V7*^0~d;3PI#{5kVXH2ifFylj4wCsL-MU;En*s)Hb%f@dn&EYr~^_}-`(x6$p=OCO*Z8 zyaIaGmSan2Vgawrryk`w#m8{;m$xeZi*-3(^Rka8_Zp7#pWOvr?oM(kC5P$M_AL=> z=TcdVzkleoc*so@i_H$8SqXR4+Ce}Sy%sD|w&t+t16WGcTBfZ;-2Fd1-k zDU*9^YpbZrUGs*MxMBIMntq1ViC4_pyL-P__H6}3exTDMP`J@MF1;cXOK67DX zeO_h%-svDg7=EU-s$!|>t(W@|lhMr9PXDcxtsmHhGzmFEy5QuIZ)#@T*P>Mi_s=kg zdl$zR?-ltx=I9o(TWFU0ijgO{`wI}ScRPmdAm-afULtS{pv%{x@dIrw-9ZT{)S_Db zJ9kR_iiYmnlY;_5AOo96)qw&k{qA%f2EgxWG!}Ecy3Q}dft)EN6t%;kg|LA5#$M~) z8jv}6<8r50RLOrj<-|H%0gYDS5vniE6f1>G4WjmjTp#=63bl3kQIm&U%ZUwM z{<}3-{@7?w&1|)B$K?e|eEY(iBpW6%sDUJ_TLsWnr&$8tS#av{tZnFga)C|r(Y*GP8xQC!C^$Ru8tK`qZ`iCdAh2y z+^B$ac4Z}G4!-c7_qp0g(Tlue_B@kg;d(0R^*W)|66gVtdkICj{qA>wt!HQ^vEp2P zSPnRIib`;bM<7Qf4dW%CnZwFl29xu0CVpJo>-^{VTmOskE3Fq#f*&+Ndr$5!>A*zb zVgWR)n#4<6_wl0mjW7vs4NDZt>;4HM=@wMXH^cuVO?O#^cLynj_~F4iQrG&^<AUaQ!0eKf#v&BIhhj}iQ9d#qxj;n?wHG9 zJ2L2cHW4jR(XL*%f-zvNQB)GzLlIM#9&i+0Dp|+Th_rO&Lc_5`z z+YH~Mv(8VEK3UgY22Rr03W(gnxPF+1?*gU|*K7asViiF)nXgaIE7o|$P$^`qFDk~1 z&Q!@`X6SQp5piac(~soiNsyOiG*U>PG@uzhvPADw8bxV`;N?jEV#ZA+uY3zb*PHeK!*CZSeJCRyDQG4L~T z=z)JpMYA5tNA9b6n{CS;<$-`SF;$GO)rz6NC4Eii)&Td;4mqX$&+u!9$(^VZMHP&z zbGU>CY3s#(BX_S~VbtaJS1U}qhPF`v^C|PO4Tnn8E~iZz1e1v?Znqfe-82W%ZdbK_ z8rewH6&i`!{u17b8no%OdrI9PxKYN}rdmn&>a7@LTv6$!Dzm7Eq;eTgMMcj*byf_v z62y~DZjOL4?xk-JmY;-5wC2jy2*{!h;Fwu?rVBL4!xVFI z+s6(=s5AaU3_9U@8Bx>r2*u*Mw7;+IyVOG8X!H*b_~nUV;+J()I|sWN#`%z>PcC`V zopfsG<$3XB(d&qn+D5xg_a}6bfl%Qtq5|cE)R+1U2d-~4vev#^be6S|Xe)LUWU&h_ z8~Ugu!?v&Og=BDSOqqJFlX$JI%j|g}YBwqC@b@~R=m_Wbw;3ooj`vvXrGJh|BSK1Q|7;Bmp9{SW4WM~+! zZRF-@?Hn)HwzKpgA2{rOhhCdDfuq~`e2%@f;eql{O{T}fj%gQqaBv5}{)-*;)GZ?f zNwf9-djLGiw)5`e`#SC`oa_KwfI zau5b>6Puc+S{>gWSn2#^nm_zZXk*4g-_I~R=(yz)N!J@_#-=1cy9iw8VNZUZ)QP4I zG1feM|ABuUP269mjpnF=h5?Gx($;r4dHyjYfEGIejX1%xlFliXmLHQ>7+o4&Mz?R` zo1t?Twb<=Gqr)RGN@m*y)yyWlQ-{FpN;#Vb)fuNPmcc^Qe{njvR%5#!bdW-#pJ1B+ zN^_Y7skU4tR?ek0^wmqo%+fD6-Q4eZ@_mH{glP3Eip&1Vf^D;spFKX9Gos4*veZ{C z}uk#=HJfU^%US!+y^flRz%J+_=GD=vUAs1Aj z^xG?aY{SckIa3bScZl0vdCGq7ybC`()M8?3xh)y6`xiUJ4bg?x3)5_y2KRpOOG5{2|fHm zN^&==%SmsFaF!RB;Wc?4Gg1<2DJgK@8}~;EqY5Kz!(<<7jD~gff!>z`Qs5iUze7Ua zs(5`>B+^AO>CKRe3Da3z?1RQh^8~{Ovx0bUcfZoWk_nI%l_KK@7}M9N%G5mAS1Y~^ z45c|mhV%VH6V0t6LRKD{S5)h_5WpnC?505SZoMt~i`)|RuH#I}D(4%z%^9+Iw8}R% zD~W?X)9ru7qLCrXr0vfMr*plXKW>v-7sJ;J2AIxjQx4>c*2_5cgwtoenQT)1PxIFw z!FR3FxXJZrX)ZLq1xao33q@yZs%Mk=ON!ot(cpFe^IW71m9+AIRyYz9_gs*f5{Uc1 z7Q1dwb)IZ7-Gza~LP_s{Yn3wvJTtDTFwJwbU%ad9%x~g5%4?Ede#yf8!xMhGXcaSC zE8aDt2|XppT2I}OiNqgiYg+_`Kvli^zxfKnhc-{ubGj~O35eU%X8N!wz8hbyQ(Y?v z<02Eob*SI8C7@e$hHl*!t#18IXh(bj5%1h!{TP|Z-nspEL&*Q^&JT}V(AQzELpPD8 z%_4-eB(O2=R^BVGV9KWT;)H*zzfOj6Fl`;VHkt=oKq)uM#`-(k4aSq1Q61aOXUuMH zZo`w4$Kjc&z-|^!GYYbu1P=D#MR8zTN3NPYuyNLO*E6%IsEDRc>dSxY^uSt;%AeKN gKjZ&as6V(QxOSu8oyD4(ivSqU6;&0== lx / 2, y < 1 * ly / 6).astype(float), + np.logical_and(np.logical_and(x < lx / 2, y >= 1 * ly / 6), y < 8 * ly / 15).astype(float), + np.logical_and(np.logical_and(x >= lx / 2, y >= 1 * ly / 6), y < 8 * ly / 15).astype(float), + np.logical_and(np.logical_and(x < 3 * lx / 8, y >= 8 * ly / 15), y < 5 * ly / 6).astype(float), + np.logical_and(np.logical_and(x >= 3 * lx / 8, x < 0.5 * lx), y >= 8 * ly / 15).astype(float), + np.logical_and(np.logical_and(x >= 0.5 * lx, x < 5 * lx / 8), y >= 8 * ly / 15).astype(float), + np.logical_and(np.logical_and(x >= 5 * lx / 8, y >= 8 * ly / 15), y < 5 * ly / 6).astype(float), + np.logical_and(x < 3 * lx / 8, y >= 5 * ly / 6).astype(float), + np.logical_and(x >= 5 * lx / 8, y >= 5 * ly / 6).astype(float) + ] + + m_true = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(len(mask_values))] + masks = [Function(V) for _ in range(len(mask_values))] + for mask, values in zip(masks, mask_values): + mask.dat.data[:] = values + + M = len(m_true) + + # Define a function to update n based on m i.e. the mapping + def update_n(n, m): + # Reset n to zero + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, masks): + n += m_ * mask_ + + update_n(manning_2d, m_true) +elif selected_case == 'IndependentPointsScheme': + # Define our values for n + points = [(0, 0), (0, 0.5 * ly), (0, ly), (0.5 * lx, 0), (lx, 0), (lx, 0.5 * ly), (lx, ly), (0.5 * lx, 0.5 * ly), + (0.1 * lx, 0.9 * ly), (0.3 * lx, 0.9 * ly), (0.7 * lx, 0.9 * ly), (0.9 * lx, 0.9 * ly), + (0.1 * lx, 0.7 * ly), (0.3 * lx, 0.7 * ly), (0.7 * lx, 0.7 * ly), (0.9 * lx, 0.7 * ly), + (0.3 * lx, 0.75 * ly), (0.7 * lx, 0.75 * ly), (0.5 * lx, 0.4 * ly), (0.5 * lx, 0.1 * ly), + (0.1 * lx, 0.25 * ly), (0.3 * lx, 0.25 * ly), (0.7 * lx, 0.25 * ly), (0.9 * lx, 0.25 * ly), + (0.1 * lx, 0.05 * ly), (0.3 * lx, 0.05 * ly), (0.7 * lx, 0.05 * ly), (0.9 * lx, 0.05 * ly)] + m_true = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(len(points))] + M = len(m_true) + + # Use Python's numpy to create arrays for the interpolation points + interp_x = np.array([p[0] for p in points]) + interp_y = np.array([p[1] for p in points]) + points = np.column_stack((interp_x, interp_y)) + + linear_interpolator = LinearNDInterpolator(points, np.eye(len(points))) + nearest_interpolator = NearestNDInterpolator(points, np.eye(len(points))) + + # Apply the interpolators to the mesh coordinates + linear_coefficients = linear_interpolator(coordinates) + nan_mask = np.isnan(linear_coefficients).any(axis=1) + linear_coefficients[nan_mask] = nearest_interpolator(coordinates[nan_mask]) + + # Create Function objects to store the coefficients + masks = [Function(V) for _ in range(len(points))] + + # Assign the linear coefficients to the masks + for i, mask in enumerate(masks): + mask.dat.data[:] = linear_coefficients[:, i] + + # Define a function to update n based on m i.e. the mapping + def update_n(n, m): + # Reset n to zero + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, masks): + n += m_ * mask_ + + update_n(manning_2d, m_true) +else: + pass + +# Setup controls and regularization parameters (regularisation only needed where lots of values are being changed) +gamma_hessian_list = [] +control_bounds_list = [] +gamma_hessian = None +bounds = [0.01, 0.06] +if selected_case == 'NodalFreedom': + gamma_hessian = Constant(1.0) +if gamma_hessian is not None: + print_output(f'Manning regularization params: hess={float(gamma_hessian):.3g}') + gamma_hessian_list.append(gamma_hessian) +control_bounds_list.append(bounds) +# reshape to [[lo1, lo2, ...], [hi1, hi2, ...]] +control_bounds = numpy.array(control_bounds_list).T + +# Assign initial conditions +print_output('Exporting to ' + options.output_directory) + +# Create station manager +observation_data_dir = output_dir_forward +variable = 'uv' +lx, ly = np.max(x), np.max(y) +stations = [ + ('stationA', (lx/10, ly/2)), + ('stationB', (lx/2, ly/2)), + ('stationC', (3*lx/4, ly/4)), + ('stationD', (3*lx/4, 3*ly/4)), + ('stationE', (9*lx/10, ly/2)), +] +sta_manager = inversion_tools.StationObservationManager(mesh2d, output_directory=options.output_directory) +print_output('Station Manager instantiated.') +station_names, observation_coords, observation_time, observation_u, observation_v = [], [], [], [], [] +for name, (sta_x, sta_y) in stations: + file = f'{output_dir_forward}/'f'diagnostic_timeseries_{name}_{variable}.hdf5' + with h5py.File(file) as h5file: + t = h5file['time'][:].flatten() + var = h5file[name][:] + station_names.append(name) + observation_coords.append((sta_x, sta_y)) + observation_time.append(t) + observation_u.append(var[:, 0]) + observation_v.append(var[:, 1]) +observation_x, observation_y = numpy.array(observation_coords).T +sta_manager.register_observation_data(station_names, variable, observation_time, observation_u, + observation_v, observation_x, observation_y, + start_times=None, end_times=None) +print_output('Data registered.') +sta_manager.construct_evaluator() +sta_manager.set_model_field(solver_obj.fields.uv_2d) +print_output('Station Manager set-up complete.') + +# -------------------------------------- Step 3: define the optimisation problem --------------------------------------- + +# Define the scaling for the cost function so that J ~ O(1) +J_scalar = Constant(solver_obj.dt / options.simulation_end_time, domain=mesh2d) + +# Create inversion manager and add controls +inv_manager = inversion_tools.InversionManager( + sta_manager, output_dir=options.output_directory, no_exports=False, penalty_parameters=gamma_hessian_list, + cost_function_scaling=J_scalar, test_consistency=False, test_gradient=False) + +print_output('Inversion Manager instantiated.') + +if selected_case == 'Constant': + manning_const = Constant(0.03, name='Manning', domain=mesh2d) + manning_2d.project(manning_const) + inv_manager.add_control(manning_const) +elif selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': + m_values = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(M)] + + # Define a function to update n based on m i.e. the mapping + def update_n(n, m): + # Reset n to zero + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, masks): + n += m_ * mask_ + + update_n(manning_2d, m_values) + for i, control in enumerate(m_values): + if i == 0: + inv_manager.add_control(control, masks[0], new_map=True) + else: + inv_manager.add_control(control, masks[i]) +else: + manning_2d.assign(0.04) + inv_manager.add_control(manning_2d) +if not no_exports: + VTKFile(output_dir_invert + '/manning_init.pvd').write(manning_2d) + +# Extract the regularized cost function +cost_function = inv_manager.get_cost_function(solver_obj, weight_by_variance=True) + +# Solve and setup reduced functional +solver_obj.iterate(update_forcings=update_forcings, export_func=cost_function) +inv_manager.stop_annotating() + +# Run inversion +opt_verbose = -1 # scipy diagnostics -1, 0, 1, 99, 100, 101 +opt_options = { + 'maxiter': 20, # NOTE increase to run iteration longer + 'ftol': 1e-5, + 'disp': opt_verbose if mesh2d.comm.rank == 0 else -1, +} +if os.getenv('THETIS_REGRESSION_TEST') is not None: + opt_options['maxiter'] = 1 +control_opt_list = inv_manager.minimize( + opt_method='L-BFGS-B', bounds=control_bounds, **opt_options) +if isinstance(control_opt_list, Function): + control_opt_list = [control_opt_list] +for oc, cc in zip(control_opt_list, inv_manager.control_coeff_list): + print_function_value_range(oc, prefix='Optimal') + # NOTE: This would need to be updated to match the controls if bathymetry was also added! + if selected_case == 'Constant': + P1_2d = get_functionspace(mesh2d, 'CG', 1) + manning_2d = Function(P1_2d, name='Manning coefficient') + manning_2d.assign(domain_constant(inv_manager.m_list[-1], mesh2d)) + if not no_exports: + VTKFile(f'{options.output_directory}/manning_optimised.pvd').write(manning_2d) + elif selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': + P1_2d = get_functionspace(mesh2d, 'CG', 1) + manning_2d = Function(P1_2d, name='manning2d') + + # Define a function to update n based on m i.e. the mapping + def update_n(n, m): + # Reset n to zero + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, masks): + n += m_ * mask_ + + update_n(manning_2d, inv_manager.m_list) + VTKFile(f'{options.output_directory}/manning_optimised.pvd').write(manning_2d) + else: + name = cc.name() + oc.rename(name) + print_function_value_range(oc, prefix='Optimal') + if not no_exports: + VTKFile(f'{options.output_directory}/{name}_optimised.pvd').write(oc) + +if selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': + print("Optimised vector m:\n", + [np.round(control_opt_list[i].dat.data[0], 4) for i in range(len(control_opt_list))]) diff --git a/examples/headland_inversion/inversion_tools_vel.py b/examples/headland_inversion/inversion_tools_vel.py new file mode 100644 index 000000000..af464eb79 --- /dev/null +++ b/examples/headland_inversion/inversion_tools_vel.py @@ -0,0 +1,797 @@ +import firedrake as fd +from firedrake.adjoint import * +import ufl +from thetis.configuration import FrozenHasTraits +from thetis.solver2d import FlowSolver2d +from thetis.utility import create_directory, print_function_value_range, get_functionspace, unfrozen, domain_constant +from thetis.log import print_output +from thetis.diagnostics import HessianRecoverer2D +from thetis.exporter import HDF5Exporter +import abc +import numpy +import h5py +from scipy.interpolate import interp1d +import time as time_mod +import os + + +def group_consecutive_elements(indices): + grouped_list = [] + current_group = [] + + for index in indices: + if not current_group or index == current_group[-1]: + current_group.append(index) + else: + grouped_list.append(current_group) + current_group = [index] + + # Append the last group + if current_group: + grouped_list.append(current_group) + + return grouped_list + + +class InversionManager(FrozenHasTraits): + """ + Class for handling inversion problems and stashing + the progress of the associated optimization routines. + """ + + @unfrozen + def __init__(self, sta_manager, output_dir='outputs', no_exports=False, real=False, + penalty_parameters=[], cost_function_scaling=None, + test_consistency=True, test_gradient=True): + """ + :arg sta_manager: the :class:`StationManager` instance + :kwarg output_dir: model output directory + :kwarg no_exports: if True, nothing will be written to disk + :kwarg real: is the inversion in the Real space? + :kwarg penalty_parameters: a list of penalty parameters to pass + to the :class:`ControlRegularizationManager` + :kwarg cost_function_scaling: global scaling for the cost function. + As rule of thumb, it's good to scale the functional to J < 1. + :kwarg test_consistency: toggle testing the correctness with + which the :class:`ReducedFunctional` can recompute values + :kwarg test_gradient: toggle testing the correctness with + which the :class:`ReducedFunctional` can recompute gradients + """ + assert isinstance(sta_manager, StationObservationManager) + self.sta_manager = sta_manager + self.reg_manager = None + self.output_dir = output_dir + self.no_exports = no_exports or real + self.real = real + self.maps = [] + self.penalty_parameters = penalty_parameters + self.cost_function_scaling = cost_function_scaling or fd.Constant(1.0) + self.sta_manager.cost_function_scaling = self.cost_function_scaling + self.test_consistency = test_consistency + self.test_gradient = test_gradient + self.outfiles_m = [] + self.outfiles_dJdm = [] + self.outfile_index = [] + self.control_exporters = [] + self.initialized = False + + self.J = 0 # cost function value (float) + self.J_reg = 0 # regularization term value (float) + self.J_misfit = 0 # misfit term value (float) + self.dJdm_list = None # cost function gradient (Function) + self.m_list = None # control (Function) + self.Jhat = None + self.m_progress = [] + self.J_progress = [] + self.J_reg_progress = [] + self.J_misfit_progress = [] + self.dJdm_progress = [] + self.i = 0 + self.tic = None + self.nb_grad_evals = 0 + self.control_coeff_list = [] + self.control_list = [] + self.control_family = [] + + def initialize(self): + if not self.no_exports: + if self.real: + raise ValueError("Exports are not supported in Real mode.") + create_directory(self.output_dir) + create_directory(self.output_dir + '/hdf5') + for i in range(max(self.outfile_index)+1): + self.outfiles_m.append( + fd.output.vtk_output.VTKFile(f'{self.output_dir}/control_progress_{i:02d}.pvd')) + self.outfiles_dJdm.append( + fd.output.vtk_output.VTKFile(f'{self.output_dir}/gradient_progress_{i:02d}.pvd')) + self.initialized = True + + def add_control(self, f, mapping=None, new_map=False): + """ + Add a control field. + + Can be called multiple times in case of multiparameter optimization. + + :arg f: Function or Constant to be used as a control variable. + :kwarg mapping: map that dictates how each Control relates to the function being altered + :kwarg new_map: if True, create a new exporter + """ + if len(self.control_coeff_list) == 0: + new_map = True + if mapping: + self.maps.append(mapping) + else: + self.maps.append(None) + self.control_coeff_list.append(f) + self.control_list.append(Control(f)) + function_space = f.function_space() + element = function_space.ufl_element() + family = element.family() + self.control_family.append(family) + if isinstance(f, fd.Function) and not self.no_exports: + j = len(self.control_coeff_list) - 1 + prefix = f'control_{j:02d}' + if family == 'Real': # i.e. Control is a Constant (assigned to a mesh, so won't register as one) + if new_map: + self.control_exporters.append( + HDF5Exporter(get_functionspace(self.sta_manager.mesh, 'CG', 1), self.output_dir + '/hdf5', + prefix) + ) + if len(self.control_coeff_list) == 1: + self.outfile_index.append(0) + else: + self.outfile_index.append(self.outfile_index[-1] + 1) + else: + self.outfile_index.append(self.outfile_index[-1]) + else: # i.e. Control is a Function + self.control_exporters.append( + HDF5Exporter(function_space, self.output_dir + '/hdf5', prefix) + ) + if len(self.control_coeff_list) == 1: + self.outfile_index.append(0) + else: + self.outfile_index.append(self.outfile_index[-1] + 1) + + def reset_counters(self): + self.nb_grad_evals = 0 + + def set_control_state(self, j, djdm_list, m_list): + """ + Stores optimization state. + + To call whenever variables are updated. + + :arg j: error functional value + :arg djdm_list: list of gradient functions + :arg m_list: list of control coefficents + """ + self.J = j + self.dJdm_list = djdm_list + self.m_list = m_list + + tape = get_working_tape() + reg_blocks = tape.get_blocks(tag="reg_eval") + self.J_reg = sum([b.get_outputs()[0].saved_output for b in reg_blocks]) + misfit_blocks = tape.get_blocks(tag="misfit_eval") + self.J_misfit = sum([b.get_outputs()[0].saved_output for b in misfit_blocks]) + + def start_clock(self): + self.tic = time_mod.perf_counter() + + def stop_clock(self): + toc = time_mod.perf_counter() + return toc + + def set_initial_state(self, *state): + self.set_control_state(*state) + self.update_progress() + + def update_progress(self): + """ + Updates optimization progress and stores variables to disk. + + To call after successful line searches. + """ + toc = self.stop_clock() + if self.i == 0: + for f in self.control_coeff_list: + print_function_value_range(f, name=f.name, prefix='Initial') + + elapsed = '-' if self.tic is None else f'{toc - self.tic:.1f} s' + self.tic = toc + + if not self.initialized: + self.initialize() + + # cost function and gradient norm output + djdm = [fd.norm(f) for f in self.dJdm_list] + if self.real: + controls = [m.dat.data[0] for m in self.m_list] + self.m_progress.append(controls) + self.J_progress.append(self.J) + self.J_reg_progress.append(self.J_reg) + self.J_misfit_progress.append(self.J_misfit) + self.dJdm_progress.append(djdm) + comm = self.control_coeff_list[0].comm + if comm.rank == 0 and not self.no_exports: + if self.real: + numpy.save(f'{self.output_dir}/m_progress', self.m_progress) + numpy.save(f'{self.output_dir}/J_progress', self.J_progress) + numpy.save(f'{self.output_dir}/J_reg_progress', self.J_reg_progress) + numpy.save(f'{self.output_dir}/J_misfit_progress', self.J_misfit_progress) + numpy.save(f'{self.output_dir}/dJdm_progress', self.dJdm_progress) + if len(djdm) > 10: + djdm = f"[{numpy.min(djdm):.4e} .. {numpy.max(djdm):.4e}]" + else: + djdm = "[" + ", ".join([f"{dj:.4e}" for dj in djdm]) + "]" + print_output(f'line search {self.i:2d}: ' + f'J={self.J:.3e}, dJdm={djdm}, ' + f'grad_ev={self.nb_grad_evals}, duration {elapsed}') + + if not self.no_exports: + grouped_indices = group_consecutive_elements(self.outfile_index) + ref_index = 0 + for j in range(len(self.outfiles_m)): + m = self.m_list[ref_index] + o_m = self.outfiles_m[j] + o_jm = self.outfiles_dJdm[j] + self.dJdm_list[ref_index].rename('Gradient') + e = self.control_exporters[j] + if len(grouped_indices[j]) > 1: + # can only be Constants + mesh2d = self.sta_manager.mesh + P1_2d = get_functionspace(mesh2d, 'CG', 1) + # vtk format + field = fd.Function(P1_2d, name='control') + self.update_n(field, self.m_list[ref_index:ref_index + len(grouped_indices[j])], + ref_index, ref_index + len(grouped_indices[j])) + o_m.write(field) + # hdf5 format + e.export(field) + # gradient output + gradient = fd.Function(P1_2d, name='Gradient') + self.update_n(gradient, self.dJdm_list[ref_index:ref_index + len(grouped_indices[j])], + ref_index, ref_index + len(grouped_indices[j])) + o_jm.write(gradient) + else: + if self.control_family[ref_index] == 'Real': + mesh2d = self.sta_manager.mesh + P1_2d = get_functionspace(mesh2d, 'CG', 1) + # vtk format + field = fd.Function(P1_2d, name='control') + field.assign(domain_constant(m, mesh2d)) + o_m.write(field) + # hdf5 format + e.export(field) + # gradient output + gradient = fd.Function(P1_2d, name='Gradient') + gradient.assign(domain_constant(self.dJdm_list[ref_index], mesh2d)) + o_jm.write(gradient) + else: + # control output + # vtk format + m.rename(self.control_coeff_list[j].name()) + o_m.write(m) + # hdf5 format + e.export(m) + # gradient output + o_jm.write(self.dJdm_list[ref_index]) + ref_index += len(grouped_indices[j]) + + self.i += 1 + self.reset_counters() + + @property + def rf_kwargs(self): + """ + Default keyword arguments to pass to the + :class:`ReducedFunctional` class. + """ + def gradient_eval_cb(j, djdm, m): + self.set_control_state(j, djdm, m) + self.nb_grad_evals += 1 + return djdm + + params = { + 'derivative_cb_post': gradient_eval_cb, + } + return params + + def get_cost_function(self, solver_obj, weight_by_variance=False): + r""" + Get a sum of square errors cost function for the problem: + + ..math:: + J(u) = \sum_{i=1}^{n_{ts}} \sum_{j=1}^{n_{sta}} (u_j^{(i)} - u_{j,o}^{(i)})^2, + + where :math:`u_{j,o}^{(i)}` and :math:`u_j^{(i)}` denote the + observed and computed values at timestep :math:`i`, and + :math:`n_{ts}` and :math:`n_{sta}` are the numbers of timesteps + and stations, respectively. + + Regularization terms are included if a + :class:`RegularizationManager` instance is provided. + + :arg solver_obj: the :class:`FlowSolver2d` instance + :kwarg weight_by_variance: should the observation data be + weighted by the variance at each station? + """ + assert isinstance(solver_obj, FlowSolver2d) + if len(self.penalty_parameters) > 0: + self.reg_manager = ControlRegularizationManager( + self.control_coeff_list, + self.penalty_parameters, + self.cost_function_scaling, + RSpaceRegularizationCalculator if self.real else HessianRegularizationCalculator) + self.J_reg = 0 + self.J_misfit = 0 + if self.reg_manager is not None: + self.J_reg = self.reg_manager.eval_cost_function() + self.J = self.J_reg + + if weight_by_variance: + var = fd.Function(self.sta_manager.fs_points_0d) + for i, j in enumerate(self.sta_manager.local_station_index): + obs_speed = numpy.sqrt( + numpy.array(self.sta_manager.observation_u[j]) ** 2 + + numpy.array(self.sta_manager.observation_v[j]) ** 2) + var.dat.data[i] = numpy.var(obs_speed) + self.sta_manager.station_weight_0d.interpolate(1 / var) + + def cost_fn(): + t = solver_obj.simulation_time + misfit = self.sta_manager.eval_cost_function(t) + self.J_misfit += misfit + self.J += misfit + + return cost_fn + + @property + def reduced_functional(self): + """ + Create a Pyadjoint :class:`ReducedFunctional` for the optimization. + """ + if self.Jhat is None: + self.Jhat = ReducedFunctional(self.J, self.control_list, **self.rf_kwargs) + return self.Jhat + + def stop_annotating(self): + """ + Stop recording operations for the adjoint solver. + + This method should be called after the :meth:`iterate` + method of :class:`FlowSolver2d`. + """ + assert self.reduced_functional is not None + if self.test_consistency: + self.consistency_test() + if self.test_gradient: + self.taylor_test() + pause_annotation() + + def get_optimization_callback(self): + """ + Get a callback for stashing optimization progress + after successful line search. + """ + + def optimization_callback(m): + self.update_progress() + if not self.no_exports: + self.sta_manager.dump_time_series() + + return optimization_callback + + def update_n(self, n, m, start_index, end_index): + """ + Update the function `n` by adding weighted masks from `m`. + + This method resets `n` to zero and then adds the weighted masks defined in + `self.maps` to `n`. Each element in `m` is multiplied by the corresponding + mask before being added to `n`. + + :param n: Function or Field to be updated. + :param m: List of Constants/Functions to be combined with the masks. + :param start_index: Starting map. + :param end_index: Final map. + :raises ValueError: If the lengths of `m` and `self.maps` do not match. + """ + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, self.maps[start_index:end_index]): + n += m_ * mask_ + + def minimize(self, opt_method="BFGS", bounds=None, **opt_options): + """ + Minimize the reduced functional using a given optimization routine. + + :kwarg opt_method: the optimization routine + :kwarg bounds: a list of bounds to pass to the optimization routine + :kwarg opt_options: other optimization parameters to pass + """ + print_output(f'Running {opt_method} optimization') + self.reset_counters() + self.start_clock() + J = float(self.reduced_functional(self.control_coeff_list)) + self.set_initial_state(J, self.reduced_functional.derivative(), self.control_coeff_list) + if not self.no_exports: + self.sta_manager.dump_time_series() + return minimize( + self.reduced_functional, method=opt_method, bounds=bounds, + callback=self.get_optimization_callback(), options=opt_options) + + def consistency_test(self): + """ + Test that :attr:`reduced_functional` can correctly recompute the + objective value, assuming that none of the controls have changed + since it was created. + """ + print_output("Running consistency test") + J = self.reduced_functional(self.control_coeff_list) + if not numpy.isclose(J, self.J): + raise ValueError(f"Consistency test failed (expected {self.J}, got {J})") + print_output("Consistency test passed!") + + def taylor_test(self): + """ + Run a Taylor test to check that the :attr:`reduced_functional` can + correctly compute consistent gradients. + + Note that the Taylor test is applied on the current control values. + """ + func_list = [] + for f in self.control_coeff_list: + dc = f.copy(deepcopy=True) + func_list.append(dc) + minconv = taylor_test(self.reduced_functional, self.control_coeff_list, func_list) + if minconv < 1.9: + raise ValueError("Taylor test failed") # NOTE: Pyadjoint already prints the testing + print_output("Taylor test passed!") + + +class StationObservationManager: + """ + Implements error functional based on observation time series. + + The functional is the squared sum of error between the model and + observations. + + This object evaluates the model fields at the station locations, + interpolates the observations time series to the model time, computes the + error functional, and also stores the model's time series data to disk. + """ + def __init__(self, mesh, output_directory='outputs'): + """ + :arg mesh: the 2D mesh object. + :kwarg output_directory: directory where model time series are stored. + """ + self.mesh = mesh + on_sphere = self.mesh.geometric_dimension() == 3 + if on_sphere: + raise NotImplementedError('Sphere meshes are not supported yet.') + self.cost_function_scaling = fd.Constant(1.0) + self.output_directory = output_directory + # keep observation time series in memory + self.obs_func_u_list = [] + self.obs_func_v_list = [] + # keep model time series in memory during optimization progress + self.station_value_u_progress = [] + self.station_value_v_progress = [] + # model time when cost function was evaluated + self.simulation_time = [] + self.model_observation_field = None + self.initialized = False + + def register_observation_data(self, station_names, variable, time, + u, v, x, y, start_times=None, end_times=None): + """ + Add station time series data to the object. + + The `x`, and `y` coordinates must be such that + they allow extraction of model data at the same coordinates. + + :arg list station_names: list of station names + :arg str variable: canonical variable name, e.g. 'elev' + :arg list time: array of time stamps, one for each station + :arg list values: array of observations, one for each station + :arg list x: list of station x coordinates + :arg list y: list of station y coordinates + :kwarg list start_times: optional start times for the observation periods + :kwarg list end_times: optional end times for the observation periods + """ + self.station_names = station_names + self.variable = variable + self.observation_time = time + self.observation_u = u + self.observation_v = v + self.observation_x = x + self.observation_y = y + num_stations = len(station_names) + self._start_times = start_times or -numpy.ones(num_stations)*numpy.inf + self._end_times = end_times or numpy.ones(num_stations)*numpy.inf + + def set_model_field(self, function): + """ + Set the model field that will be evaluated. + """ + self.model_observation_field = function + + def update_stations_in_use(self, t): + """ + Indicate which stations are in use at the current time. + + An entry of unity indicates use, whereas zero indicates disuse. + """ + if not hasattr(self, 'obs_start_times'): + self.construct_evaluator() + in_use = fd.Function(self.fs_points_0d) + in_use.dat.data[:] = numpy.array( + numpy.bitwise_and( + self.obs_start_times <= t, t <= self.obs_end_times + ), dtype=float) + self.indicator_0d.assign(in_use) + + def construct_evaluator(self): + """ + Builds evaluators needed to compute the error functional. + """ + # Create 0D mesh for station evaluation + xy = numpy.array((self.observation_x, self.observation_y)).T + mesh0d = fd.VertexOnlyMesh(self.mesh, xy) + self.fs_points_0d = fd.FunctionSpace(mesh0d, 'DG', 0) + self.obs_values_0d_u = fd.Function(self.fs_points_0d, name='u observations') + self.obs_values_0d_v = fd.Function(self.fs_points_0d, name='v observations') + self.mod_values_0d_u = fd.Function(self.fs_points_0d, name='u model values') + self.mod_values_0d_v = fd.Function(self.fs_points_0d, name='v model values') + self.indicator_0d = fd.Function(self.fs_points_0d, name='station use indicator') + self.indicator_0d.assign(1.0) + self.cost_function_scaling_0d = fd.Constant(0.0, domain=mesh0d) + self.cost_function_scaling_0d.assign(self.cost_function_scaling) + self.station_weight_0d = fd.Function(self.fs_points_0d, name='station-wise weighting') + self.station_weight_0d.assign(1.0) + interp_kw = {} + if numpy.isfinite(self._start_times).any() or numpy.isfinite(self._end_times).any(): + interp_kw.update({'bounds_error': False, 'fill_value': 0.0}) + + # Construct timeseries interpolator + self.station_interpolators_u = [] + self.station_interpolators_v = [] + self.local_station_index = [] + for i in range(self.fs_points_0d.dof_dset.size): + # loop over local DOFs and match coordinates to observations + # NOTE this must be done manually as VertexOnlyMesh reorders points + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + xy_diff = xy - numpy.array([x_mesh, y_mesh]) + xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) + j = numpy.argmin(xy_dist) + self.local_station_index.append(j) + + x, y = xy[j, :] + t = self.observation_time[j] + u = self.observation_u[j] + v = self.observation_v[j] + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + + msg = 'bad station location ' \ + f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' + assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg + # create temporal interpolator + ip_u = interp1d(t, u, **interp_kw) + ip_v = interp1d(t, v, **interp_kw) + self.station_interpolators_u.append(ip_u) + self.station_interpolators_v.append(ip_v) + + # Process start and end times for observations + self.obs_start_times = numpy.array([ + self._start_times[i] for i in self.local_station_index + ]) + self.obs_end_times = numpy.array([ + self._end_times[i] for i in self.local_station_index + ]) + + # expressions for cost function + self.misfit_expr_u = self.obs_values_0d_u - self.mod_values_0d_u + self.misfit_expr_v = self.obs_values_0d_v - self.mod_values_0d_v + self.misfit_expr = (self.misfit_expr_u ** 2 + self.misfit_expr_v ** 2) ** (1/2) + self.initialized = True + + def eval_observation_at_time(self, t): + """ + Evaluate observation time series at the given time. + + :arg t: model simulation time + :returns: list of observation time series values at time `t` + """ + self.update_stations_in_use(t) + ip1 = [float(ip(t)) for ip in self.station_interpolators_u] + ip2 = [float(ip(t)) for ip in self.station_interpolators_v] + return ip1, ip2 + + def eval_cost_function(self, t): + """ + Evaluate the cost function. + + Should be called at every export of the forward model. + """ + assert self.initialized, 'Not initialized, call construct_evaluator first.' + assert self.model_observation_field is not None, 'Model field not set.' + self.simulation_time.append(t) + # evaluate observations at simulation time and stash the result + obs_func_u, obs_func_v = fd.Function(self.fs_points_0d), fd.Function(self.fs_points_0d) + obs_func_u.dat.data[:], obs_func_v.dat.data[:] = self.eval_observation_at_time(t) + self.obs_func_u_list.append(obs_func_u) + self.obs_func_v_list.append(obs_func_v) + + # compute square error + self.obs_values_0d_u.assign(obs_func_u) + self.obs_values_0d_v.assign(obs_func_v) + P1_2d = get_functionspace(self.mesh, 'CG', 1) + u_mod = fd.Function(P1_2d, name='u velocity') + v_mod = fd.Function(P1_2d, name='v velocity') + u_mod.project(self.model_observation_field[0]) + v_mod.project(self.model_observation_field[1]) + self.mod_values_0d_u.interpolate(u_mod, ad_block_tag='u observation') + self.mod_values_0d_v.interpolate(v_mod, ad_block_tag='v observation') + s = self.cost_function_scaling_0d * self.indicator_0d * self.station_weight_0d + self.J_misfit = fd.assemble(s * self.misfit_expr ** 2 * fd.dx, ad_block_tag='misfit_eval') + return self.J_misfit + + def dump_time_series(self): + """ + Stores model time series to disk. + + Obtains station time series from the last optimization iteration, + and stores the data to disk. + + The output files are have the format + `{odir}/diagnostic_timeseries_progress_{station_name}_{variable}.hdf5` + + The file contains the simulation time in the `time` array, and the + station name and coordinates as attributes. The time series data is + stored as a 2D (n_iterations, n_time_steps) array. + """ + assert self.station_names is not None + + create_directory(self.output_directory) + tape = get_working_tape() + blocks_u = tape.get_blocks(tag='u observation') + blocks_v = tape.get_blocks(tag='v observation') + ts_data_u = [b.get_outputs()[0].saved_output.dat.data for b in blocks_u] + ts_data_v = [b.get_outputs()[0].saved_output.dat.data for b in blocks_v] + # shape (ntimesteps, nstations) + ts_data_u = numpy.array(ts_data_u) + ts_data_v = numpy.array(ts_data_v) + # append + self.station_value_u_progress.append(ts_data_u) + self.station_value_v_progress.append(ts_data_v) + var = self.variable + for ilocal, iglobal in enumerate(self.local_station_index): + name = self.station_names[iglobal] + # collect time series data, shape (niters, ntimesteps) + ts_u = numpy.array([s[:, ilocal] for s in self.station_value_u_progress]) + ts_v = numpy.array([s[:, ilocal] for s in self.station_value_v_progress]) + fn = f'diagnostic_timeseries_progress_{name}_{var}.hdf5' + fn = os.path.join(self.output_directory, fn) + with h5py.File(fn, 'w') as hdf5file: + hdf5file.create_dataset('u', data=ts_u) + hdf5file.create_dataset('v', data=ts_v) + hdf5file.create_dataset('time', data=numpy.array(self.simulation_time)) + attrs = { + 'x': self.observation_x[iglobal], + 'y': self.observation_y[iglobal], + 'location_name': name, + } + hdf5file.attrs.update(attrs) + + +class RegularizationCalculator(abc.ABC): + """ + Base class for computing regularization terms. + + A derived class should set :attr:`regularization_expr` in + :meth:`__init__`. Whenever the cost function is evaluated, + the ratio of this expression and the total mesh area will + be added. + """ + @abc.abstractmethod + def __init__(self, function, scaling=1.0): + """ + :arg function: Control :class:`Function` + """ + self.scaling = scaling + self.regularization_expr = 0 + self.mesh = function.function_space().mesh() + # calculate mesh area (to scale the cost function) + self.mesh_area = fd.assemble(fd.Constant(1.0, domain=self.mesh) * fd.dx) + self.name = function.name() + + def eval_cost_function(self): + expr = self.scaling * self.regularization_expr / self.mesh_area * fd.dx + return fd.assemble(expr, ad_block_tag="reg_eval") + + +class HessianRegularizationCalculator(RegularizationCalculator): + r""" + Computes the following regularization term for a cost function + involving a control :class:`Function` :math:`f`: + + .. math:: + J = \gamma \| (\Delta x)^2 H(f) \|^2, + + where :math:`H` is the Hessian of field :math:`f`. + """ + def __init__(self, function, gamma, scaling=1.0): + """ + :arg function: Control :class:`Function` + :arg gamma: Hessian penalty coefficient + """ + super().__init__(function, scaling=scaling) + # solvers to evaluate the gradient of the control + P1v_2d = get_functionspace(self.mesh, "CG", 1, vector=True) + P1t_2d = get_functionspace(self.mesh, "CG", 1, tensor=True) + gradient_2d = fd.Function(P1v_2d, name=f"{self.name} gradient") + hessian_2d = fd.Function(P1t_2d, name=f"{self.name} hessian") + self.hessian_calculator = HessianRecoverer2D( + function, hessian_2d, gradient_2d) + + h = fd.CellSize(self.mesh) + # regularization expression |hessian|^2 + # NOTE this is normalized by the mesh element size + # d^2 u/dx^2 * dx^2 ~ du^2 + self.regularization_expr = gamma * fd.inner(hessian_2d, hessian_2d) * h**4 + + def eval_cost_function(self): + self.hessian_calculator.solve() + return super().eval_cost_function() + + +class RSpaceRegularizationCalculator(RegularizationCalculator): + r""" + Computes the following regularization term for a cost function + involving a control :class:`Function` :math:`f` from an R-space: + + .. math:: + J = \gamma (f - f_0)^2, + + where :math:`f_0` is a prior, taken to be the initial value of + :math:`f`. + """ + def __init__(self, function, gamma, eps=1.0e-03, scaling=1.0): + """ + :arg function: Control :class:`Function` + :arg gamma: penalty coefficient + :kwarg eps: tolerance for normalising by near-zero priors + """ + super().__init__(function, scaling=scaling) + R = function.function_space() + if R.ufl_element().family() != "Real": + raise ValueError("function must live in R-space") + prior = fd.Function(R, name=f"{self.name} prior") + prior.assign(function, annotate=False) # Set the prior to the initial value + self.regularization_expr = gamma * (function - prior) ** 2 / ufl.max_value(abs(prior), eps) + # NOTE: If the prior is small then dividing by prior**2 puts too much emphasis + # on the regularization. Therefore, we divide by abs(prior) instead. + + +class ControlRegularizationManager: + """ + Handles regularization of multiple control fields + """ + def __init__(self, function_list, gamma_list, penalty_term_scaling=None, + calculator=HessianRegularizationCalculator): + """ + :arg function_list: list of control functions + :arg gamma_list: list of penalty parameters + :kwarg penalty_term_scaling: Penalty term scaling factor + :kwarg calculator: class used for obtaining regularization + """ + self.reg_calculators = [] + assert len(function_list) == len(gamma_list), \ + 'Number of control functions and parameters must match' + self.reg_calculators = [ + calculator(f, g, scaling=penalty_term_scaling) + for f, g in zip(function_list, gamma_list) + ] + + def eval_cost_function(self): + return sum([r.eval_cost_function() for r in self.reg_calculators]) diff --git a/examples/headland_inversion/model_config.py b/examples/headland_inversion/model_config.py new file mode 100644 index 000000000..f67197d9c --- /dev/null +++ b/examples/headland_inversion/model_config.py @@ -0,0 +1,186 @@ +from thetis import * +from firedrake.output.vtk_output import VTKFile + + +def generate_bathymetry(mesh, H, output_directory, exporting=True): + """ + Generates a bathymetry function for a given mesh file by solving the Eikonal + equation and projects the result onto a bathymetry function. + + Parameters: + ----------- + mesh_file : str + Path to the mesh file used to create the computational mesh. + H : float + The depth value used for the bathymetry, reduces to 5 at the shoreline (ID = 3). + output_directory : str + Directory where the output files (distance and bathymetry) will be saved. + + Returns: + -------- + bathymetry : firedrake.Function + The computed bathymetry function. + """ + + # create function space + V = FunctionSpace(mesh, 'CG', 1) + + print_output("Calculating distance for bathymetry gradient") + + # Boundary + bcs = [DirichletBC(V, 0.0, 3)] # 3 = coasts PhysID + + L = 500 # distance to shore + v = TestFunction(V) + u = Function(V) + + solver_parameters = { + 'snes_type': 'ksponly', + 'ksp_rtol': 1e-4, + 'ksp_type': 'preonly', + 'pc_type': 'lu', + 'pc_factor_mat_solver_packages': 'mumps' + } + + # Before we solve the Eikonal equation, let's solve a Laplace equation to + # generate an initial guess + F = L ** 2 * (inner(grad(u), grad(v))) * dx - v * dx + solve(F == 0, u, bcs, solver_parameters=solver_parameters) + # Relax the solver parameters for the Eikonal equation + solver_parameters.update({ + 'snes_type': 'newtonls', + 'ksp_rtol': 1e-4, + 'ksp_type': 'preonly', + 'pc_type': 'lu', + 'pc_factor_mat_solver_packages': 'mumps' + }) + + epss = [500., 200., 100., 50., 20.] + + for i, eps in enumerate(epss): + print_output("Solving Eikonal with eps == " + str(float(eps))) + F = inner(sqrt(inner(grad(u), grad(u))), v) * dx - v * dx + eps * inner(grad(u), grad(v)) * dx + solve(F == 0, u, bcs, solver_parameters=solver_parameters) + + if exporting: + VTKFile(output_directory + "/dist.pvd").write(u) + + """ + Adding viscocity sponge + """ + + bathymetry = Function(V, name="bathymetry") + bathymetry.project(conditional(ge(u, L), H, (H - 5) * (u / L) + 5.)) + if exporting: + VTKFile(output_directory + '/bathymetry.pvd').write(bathymetry) + + return bathymetry + + +def construct_solver(store_station_time_series=True, **model_options): + mesh2d = Mesh('headland.msh') + pwd = os.path.abspath(os.path.dirname(__file__)) + output_directory = model_options.get('output_directory', f'{pwd}/outputs_forward') + exporting = not model_options.get('no_exports', False) + + tidal_amplitude = 1. + tidal_period = 12.42 * 60 * 60 + t_end = tidal_period + H = 40 + dt = 800. + t_export = dt + + if os.getenv('THETIS_REGRESSION_TEST') is not None: + t_end = 5*dt + + # bathymetry + P1_2d = get_functionspace(mesh2d, 'CG', 1) + bathymetry_2d = Function(P1_2d, name="bathymetry") + bathy = generate_bathymetry(mesh2d, H, output_directory, exporting) + bathymetry_2d.project(bathy) + + x, y = SpatialCoordinate(mesh2d) + lx, ly = np.max(mesh2d.coordinates.dat.data[:, 0]), np.max(mesh2d.coordinates.dat.data[:, 1]) + + # friction Manning coefficient + manning_2d = Function(P1_2d, name='Manning coefficient') + manning_low = 0.02 + manning_high = 0.05 + manning_2d.interpolate( + conditional(x < 11e3, manning_high, manning_low)) + if exporting: + VTKFile(output_directory + '/manning_init.pvd').write(manning_2d) + + # viscosity + h_viscosity = Function(P1_2d).interpolate(conditional(le(x, 1e3), 0.1*(1e3 - x), 1.0)) + if exporting: + VTKFile(output_directory + '/h_viscosity.pvd').write(h_viscosity) + + # create solver + solver_obj = solver2d.FlowSolver2d(mesh2d, bathymetry_2d) + options = solver_obj.options + + options.simulation_export_time = t_export + options.simulation_end_time = t_end + options.output_directory = output_directory + options.check_volume_conservation_2d = True + options.fields_to_export = ['uv_2d', 'elev_2d'] + options.element_family = 'dg-cg' + options.swe_timestepper_type = 'CrankNicolson' + options.swe_timestepper_options.implicitness_theta = 0.6 + options.swe_timestepper_options.solver_parameters = {'snes_rtol': 1e-9, + 'ksp_type': 'preonly', + 'pc_type': 'lu', + 'pc_factor_mat_solver_type': 'mumps', + 'mat_type': 'aij' + } + options.horizontal_viscosity = h_viscosity + options.timestep = dt + options.update(model_options) + + options.manning_drag_coefficient = manning_2d + solver_obj.add_new_field(manning_2d, 'manning_2d', + manning_2d.name(), + 'Manning2d', unit='s m-1/3') + + solver_obj.create_equations() + + if store_station_time_series: + # store elevation time series at stations + stations = [ + ('stationA', (lx/10, ly/2)), + ('stationB', (lx/2, ly/2)), + ('stationC', (3*lx/4, ly/4)), + ('stationD', (3*lx/4, 3*ly/4)), + ('stationE', (9*lx/10, ly/2)), + ] + for name, (sta_x, sta_y) in stations: + cb = DetectorsCallback(solver_obj, [(sta_x, sta_y)], ['uv_2d'], name='timeseries_'+name+'_uv', + detector_names=[name], append_to_log=False) + solver_obj.add_callback(cb) + + left_tag = 1 + right_tag = 2 + coasts_tag = 3 + tidal_elev = Function(get_functionspace(mesh2d, "CG", 1), name='tidal_elev') + tidal_elev_bc = {'elev': tidal_elev} + # noslip currently doesn't work (vector Constants are broken in firedrake adjoint) + freeslip_bc = {'un': Constant(0.0)} + solver_obj.bnd_functions['shallow_water'] = { + left_tag: tidal_elev_bc, + right_tag: tidal_elev_bc, + coasts_tag: freeslip_bc + } + + # a function to update the tidal_elev bc value every timestep + g = 9.81 + omega = 2 * pi / tidal_period + + def update_forcings(t): + print_output("Updating tidal elevation at t = {}".format(t)) + tidal_elev.project(tidal_amplitude * sin(omega * t + omega / pow(g * H, 0.5) * x)) + + # set initial condition for elevation, piecewise linear function + solver_obj.assign_initial_conditions(uv=as_vector((1e-7, 0.0)), elev=tidal_elev) + + return solver_obj, update_forcings diff --git a/examples/headland_inversion/plot_velocity_progress.py b/examples/headland_inversion/plot_velocity_progress.py new file mode 100644 index 000000000..2415b780c --- /dev/null +++ b/examples/headland_inversion/plot_velocity_progress.py @@ -0,0 +1,77 @@ +import h5py +import matplotlib +import matplotlib.pyplot as plt +import argparse + +matplotlib.rc('font', size=18) + +# parse user input +parser = argparse.ArgumentParser( + description='Plot velocity time series progress.', +) +parser.add_argument('-s', '--station', nargs='+', + help='Station(s) to plot. Multiple can be defined.', + required=True, + choices=['stationA', 'stationB', 'stationC', 'stationD', 'stationE'] + ) +parser.add_argument('--case', nargs='+', + help='Method of Manning field representation', + choices=['Constant', 'Regions', 'IndependentPointsScheme', 'NodalFreedom'], + default=['IndependentPointsScheme'], + ) +args = parser.parse_args() +station_names = sorted(args.station) + +station_str = '-'.join(station_names) + +case_to_output_dir = { + 'Constant': 'constant_friction', + 'Regions': 'region_based', + 'IndependentPointsScheme': 'independent_points_scheme', + 'NodalFreedom': 'full_nodal_flexibility' +} + +selected_case = args.case[0] +output_dir_forward = 'outputs//outputs_forward' +output_dir_invert = 'outputs/outputs_inverse/' + case_to_output_dir[selected_case] + +niter = 0 +nplots = len(station_names) + +for i, sta in enumerate(station_names): + fig, ax = plt.subplots(2, 1, figsize=(12, 10)) + + f = output_dir_forward + f'/diagnostic_timeseries_{sta}_uv.hdf5' + with h5py.File(f, 'r') as h5file: + time = h5file['time'][:].flatten() + var = h5file[sta][:] + u_vals = var[:, 0] + v_vals = var[:, 1] + + g = output_dir_invert + f'/diagnostic_timeseries_progress_{sta}_uv.hdf5' + with h5py.File(g, 'r') as h5file: + iter_times = h5file['time'][:].flatten() + u_iter_vals = h5file['u'][:] + v_iter_vals = h5file['v'][:] + + niter = u_iter_vals.shape[0] - 1 + + ax[0].plot(time, u_vals, 'k:', zorder=3, label='observation', lw=1.3) + ax[1].plot(time, v_vals, 'k:', zorder=3, label='observation', lw=1.3) + for j, u in enumerate(u_iter_vals): + ax[0].plot(iter_times, u, label=f'iteration {j}', lw=0.5) + for j, v in enumerate(v_iter_vals): + ax[1].plot(iter_times, v, lw=0.5) + ax[0].set_title(f'{sta}', size='small') + ax[0].set_ylabel('u component of velocity (m/s)') + ax[1].set_ylabel('v component of velocity (m/s)') + ax[0].grid(True), ax[1].grid(True) + ax[0].legend(ncol=1, prop={'size': 10}) + ax[1].set_xlabel('Time (s)') + + ax[0].text(0.5, 1.1, f'Optimizing Manning, {niter} iterations', ha='center', transform=ax[0].transAxes) + + # plt.show() + imgfile = f'{output_dir_invert}/optimization_progress_{selected_case}_{station_str}_ts.png' + print(f'Saving to {imgfile}') + plt.savefig(imgfile, dpi=200, bbox_inches='tight') diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index 1a53238e0..d4790a7f6 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -39,6 +39,8 @@ 'tidalfarm/tidalfarm.py', 'tidal_barrage/plotting.py', 'channel_inversion/plot_elevation_progress.py', + 'headland_inversion/inversion_tools_vel.py', + 'headland_inversion/plot_velocity_progress.py', 'tohoku_inversion/okada.py', 'tohoku_inversion/plot_convergence.py', 'tohoku_inversion/plot_elevation_initial_guess.py', diff --git a/test_adjoint/examples/test_examples.py b/test_adjoint/examples/test_examples.py index 84658fbf6..5c830e705 100644 --- a/test_adjoint/examples/test_examples.py +++ b/test_adjoint/examples/test_examples.py @@ -15,7 +15,8 @@ # list of all adjoint examples to run adjoint_files = [ 'tidalfarm/tidalfarm.py', - # 'channel_inversion/inverse_problem.py', # FIXME requires obs time series + 'channel_inversion/inverse_problem.py', + 'headland_inversion/inverse_problem.py', ] cwd = os.path.abspath(os.path.dirname(__file__)) diff --git a/thetis/utility.py b/thetis/utility.py index f787f8f5d..d57f20f61 100755 --- a/thetis/utility.py +++ b/thetis/utility.py @@ -136,6 +136,15 @@ def __setattr__(self, key, value): super(FieldDict, self).__setattr__(key, value) +def domain_constant(value, mesh, **kwargs): + """Create constant over a domain + Returns what used to be Constant(mesh, domain=mesh)""" + R = FunctionSpace(mesh, "R", 0) + c = Function(R, **kwargs) + c.assign(value) + return c + + def get_functionspace(mesh, h_family, h_degree, v_family=None, v_degree=None, vector=False, tensor=False, hdiv=False, variant=None, v_variant=None, **kwargs): From 6b85a39d0e0a7ee1db4dbfd71437f2a7cb293c88 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 19 Jul 2024 15:58:26 +0100 Subject: [PATCH 03/15] lint fix --- examples/headland_inversion/inversion_tools_vel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/headland_inversion/inversion_tools_vel.py b/examples/headland_inversion/inversion_tools_vel.py index af464eb79..1e8e98e2a 100644 --- a/examples/headland_inversion/inversion_tools_vel.py +++ b/examples/headland_inversion/inversion_tools_vel.py @@ -333,8 +333,8 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): var = fd.Function(self.sta_manager.fs_points_0d) for i, j in enumerate(self.sta_manager.local_station_index): obs_speed = numpy.sqrt( - numpy.array(self.sta_manager.observation_u[j]) ** 2 + - numpy.array(self.sta_manager.observation_v[j]) ** 2) + numpy.array(self.sta_manager.observation_u[j]) ** 2 + + numpy.array(self.sta_manager.observation_v[j]) ** 2) var.dat.data[i] = numpy.var(obs_speed) self.sta_manager.station_weight_0d.interpolate(1 / var) From f8d5cdfac39fa0152d891115b8f74e90bd4dfc9e Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 19 Jul 2024 15:59:17 +0100 Subject: [PATCH 04/15] VTKFile path --- examples/headland_inversion/inversion_tools_vel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/headland_inversion/inversion_tools_vel.py b/examples/headland_inversion/inversion_tools_vel.py index 1e8e98e2a..9d61e6eb8 100644 --- a/examples/headland_inversion/inversion_tools_vel.py +++ b/examples/headland_inversion/inversion_tools_vel.py @@ -101,9 +101,9 @@ def initialize(self): create_directory(self.output_dir + '/hdf5') for i in range(max(self.outfile_index)+1): self.outfiles_m.append( - fd.output.vtk_output.VTKFile(f'{self.output_dir}/control_progress_{i:02d}.pvd')) + fd.VTKFile(f'{self.output_dir}/control_progress_{i:02d}.pvd')) self.outfiles_dJdm.append( - fd.output.vtk_output.VTKFile(f'{self.output_dir}/gradient_progress_{i:02d}.pvd')) + fd.VTKFile(f'{self.output_dir}/gradient_progress_{i:02d}.pvd')) self.initialized = True def add_control(self, f, mapping=None, new_map=False): From 19a96cf658ed4bda55ea78c9629c6f0af3e63764 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 19 Jul 2024 16:59:58 +0100 Subject: [PATCH 05/15] Add timeseries for calibration --- .../diagnostic_timeseries_stationA_uv.hdf5 | Bin 0 -> 23464 bytes .../diagnostic_timeseries_stationB_uv.hdf5 | Bin 0 -> 23464 bytes .../diagnostic_timeseries_stationC_uv.hdf5 | Bin 0 -> 23464 bytes .../diagnostic_timeseries_stationD_uv.hdf5 | Bin 0 -> 23464 bytes .../diagnostic_timeseries_stationE_uv.hdf5 | Bin 0 -> 23464 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationA_uv.hdf5 create mode 100644 examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationB_uv.hdf5 create mode 100644 examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationC_uv.hdf5 create mode 100644 examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationD_uv.hdf5 create mode 100644 examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationE_uv.hdf5 diff --git a/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationA_uv.hdf5 b/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationA_uv.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..c1f184483d57f4806cdc14a706a21be286024ace GIT binary patch literal 23464 zcmeI2X;2hL6o6-UK{l)hXsVUzCPc#d@NFTzGkd11&;Qvj5CvP zW_==hp|(1!nHP%aUl6mMA`*(sby#h6>uZ*<)(>Ube+Di~9Bl-g@b_PdQ0>>>2ftsX z?0H|m`t6y0)4t-{o9`<={u_OzG+~<6wKV1a6@8`8zZclO@$ZkpoUA`1;0CL2)PVDq z`)u^~*tAyR6V#s{q23vq78D3HTZC?uKXOZmhpD$ZF_^Q{q}3Zqq8{^gyyXg7hSO79 zpdw^+-7D(x{x$C_fw~(1f8ohV`!9dxNvS_zioMt9-ShV_U=4<701co4G=K)s02)98 z{}ThTGWsB2Qx+a)aSEj@o#nr$RAsaIbClA2N*SRvE}|?frgSQyRFzUHFH&+9l<}7+ zrB^63Dk)`ElpCrkjcX{MG8Wd-IHr!$sh-lHfl|eo+(={PHA?d)O0Jo*oH4$I#_m5* zO0QFPFlMyUIJAvYc7sw&MY)0T3ZwB&nm@q!lyOx%%@;CGxJBa_#`}y;w`o3$(clh^ z!x>eK93E%@4WI!ufCkV28bAYR01co4G=K)s02)98XaEhM0W^RH&;S}h184vZpaC@S zwi@`W-;E~E9bk-oh|0i!#5-UShG+l{paC?12G9T+Km-3(0}Iph6ML?0x8m==;<+I1 zU7KUD@mrG+$*q}Cn=s9Ag+~TCWa&Atr_dg@M@?}ol9Uh&(e&_!J5KQP;s=XLGA@u> zht39xt25{<7!e!rsEiDsQ{uC}*bT& zU?nMPv)flgI(;1u%GWG~p9G~>hQLR>bS*F`My=Fz@nbyh#vRdXmx;g^L5!1 z#PuZC_@1{>rUT>%yB{Sc*O6l(I*RZLN624Nsbi$5BNq~OegTGVa4$j8&eSXiZas z>;B1y%a4|mxRaJnD>H*&ll=z~MHVGQ>K{F$YtUA>zhvjEjwOWLOD|2&y1NYwM~0u{ zY_rMwqpP**wZkED&*-VkjwKOyE8RKv`VkQ2Kk=UR%s_BXSSpAt42Q9!748e#Q{Y9* zqUeQWJFMC)R84$&7B)?7JKWqD0>eL!cy`{e7}k#z%{eU#fGwE|11v@5PgQI zHGu1)Ll1l9Ystfk^6%!KXoQpk>%g$<-sI)Vj`J1IufeFmoT#$dUgT1Hqh8jKW{524 z_4Y#&HVi2o{oZxtk+-IaQ$%LfwW zJp~K9FTt0kO=;tkd|}M^N{^e}r7%{zGxx-<_2AqPX4Dj02#t45%7kefz-^w*FN$5+ zaHc9OebI715Z89!9HTrA6K&QxEN|Tiz2ag153TnRRebN}5mrj@Q_Waf+>k+Dp8D0a zlLSIf$5q?FwgNJ@OX{L84S{N5N|M2pQj)lPL9|uacHs9R5)GgMG=K)s02)98XaEhM k0W^RH&;S}h184vZpaC?12G9T+Km%w14WI!u@MjJD27c>{;{X5v literal 0 HcmV?d00001 diff --git a/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationB_uv.hdf5 b/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationB_uv.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..8f3b34c55e15eeaecdbafa53d9b07a95c44df5d5 GIT binary patch literal 23464 zcmeI2X;2hL6o6-UA#OMV#Ea5WR!h`GsH}>JXRkpmqH+W=K>-y71quWL5=2D>70~iT z3q|9B5m5|>S4yKkjs;f)xkMHLSwI)OqM{KHBiW^UQYosmO!6a#Z+^_z?{&{(=HvCw zpNX@ZI^9sOpB|?@#9~f|GwvGlr%raHxX}zB^YaVYK)@Kwm?ESDfu^E%&XnbiS^d}# zXd;8xpbuq%6N38RAb84I~k-vDpzI3(0F z)HiV1=X{>;Pa8h*juddZUHwZ>+72Dwb)`T^D~@Az`1*X`^07$M`P#9GgmAQXWt^2h zXVod97wT%V+IgXf{sl4HDI%fBrU$F7X?^Vy*81Tr`_EvugX2^IC;a_aB3$?F_rdR1 z347kRuYP}K-?gv!_U8MFkN-wr1&w5yHMO+m{uO;?!oL^Tz47l415VbJ5pV{Y8#U#Y z_6s77cn?Fw}xkxDkN@+G_Rt}|;oKlrX8FYn`yGpse zfKpOOnNdV3E2i`*p_G55 zuVPd&a(JKtG=K)s02)98XaEhM0W^RH&;S}h184vZpaC?12G9T+Km%w14WI!ufCkXO z`)c5?em9yjp_?)GAu0p^5pTCe7@`3*fCkV28bAYR01f zpV^TCk^ScjHq3DYbx3e)aakU;HB0R+x4A*WSnC0=jEX?Awk~;&-vrQ)6VG{3TL!IB z>Ugp_7@j3u={-KF5)Rk+1a};bhUvBI+he0|!HMr)-uEIqz%MavxlCCL`4g?nTFxGT zw7y^Z*qEyzDYACIs^}OPw~i2o)~mogdtLjheHkG2ALY{-UJv>f(^GDLaTalt(^2aKgIHj=7pcoEp??Cp~aWA-_;k5nnap`oC-;ZZI` zS$zBP+^7nOh%4$pW3C*w#5ioRl$XMb)t-A+Wy#^y_LKb<&ASHU4l0#Hr{zJQ@cR7V z4LMMnI&os7c|LUZDt+)cITJ$iTxa%$EAYU2*@~w}caklU^9HtTEdYtAWAi$sBQ_X_K7K-GnNORBv}iOmq}!BY2IAldZ1&nq(}G2J9NoEldJ zM~(d4kNee;$O&nao_<&aLVv#(si6&I>rdO)Z?xP61qFG%(@PpjTEMiDXtQ{vsE!)oK8@N;)a)DMY|YUL?#V#LGE?Thb{gPX#V-A?}q&Sw@G9-5*i zX_j_pK1sR^RW&`As5{i;=?T-9G3so1KL4)A&Sg!+U{XM?#g1IC$PDvJ-`_+mUQd2< zL@ozM!~9y?=q9pf*!smS4SCRSin;xy!A->6p~KLzJRc&fbY^|JT1^rTkowY?D`0of z(MX(dmt>arEgGt)fT?CLw|OkTL!LMlHnzJfz-i{mH9gH5iSy+K+m$g2h+7%qE>5f` zYDt`%=Pm_Q8?PH{Ur|S9nKX|Ik5fRzu?^$)533>GO7XzafeJ88%$#EInUeVD%UVNS z6(D}?SaJMuIdM{iFSPPjK;UpM&)v%F#OHZij|jN}hHo|-Z96`XM7Q-n?RV}f=>42* zA7FWwgtfX3=n;JlD#jG{sW`nGR$s6m_OBd0yYVy*+%xoyawTG^32lV%)=WFZ|0G!?aTrYzr?)eq67 ziDNA9ICcDZPP3A)&y!7avJP}4<`c+|%7D%IuN<@wJe@tY8KaW*84EdI56}79v5)UO zUysFJR(ziCPdT6XKnggWw*I9jZHEqMyHX&e6-To=e0{!e`B3qo$mj z$NWXE&R+98=litfjWjz$+kygtc8k!B@<;kDa~|fZNk}*wZCbOD^fhC?j;n_!EyH!z zSfC;lblp4Z@&2{%D}kmO|9|1hnYCU1&Xc#a-4uJT(YxpGq1_q`(Eu7i184vZpaC?1 z2L2}oHYwpR%@q()&CmcY!jz zkkafD<*_14#T82TVoF&FWea0&DUCy}QaY$8P0J{28I#Lt>|H@AucYMEluE|%DjH9( zrZlUetY!X|Wk6c+Z6R!2HOFAC$oDda(I)ilMRiDjeu?)AO-&FzC8ZJ{-8C3=RhlQZ~E^1yk!=!kEhyP^ezGMLb~} zgf}=w+sCP)xz$+TEiMe?1`*aFFRP$Qns_iu76x7evbsf;RzsCv{X}=GZ7}kl@gU1# zH6SrP)DV{u2KK6Pp((N&&^^=jdgIdVP+7iZ_mzj$FrabT=iBASul6~l_B~{R`_cz?$S4ZGheAZj#DK*%g8}l}?@)!iy zb=Q+sRzhQ~Z*gSgaR~Y3r+($zSLv(!p?HY1W-4 z6<8WgQg*9Nh3zl83e5*zh04)JP5B1N;5fy5WKpjYaLSIDV7T`nsGn|0B`1oYW|Xdr z-O)Js?6gy=G(!nvY-jg<j{XTj9W{G>K^DVe(5^jOFnD8=+7B+G`0H zPm>uNH(2HQ`9YV_1+Rs&0+LzS!9Cr_o=o*Esm|(siHN65zP=k1K=^%#L<4954WI!u sfCkV28bAYR01co4G=K)s02)98XaEhM0W^RH&;S}h184vZ{8G zF~SlMh+ObP!esTqo90ZE=}5#Ukgt>hn=8WX# zJX4}Ks>rj7d7%>h3nI2tl!Qw2)mUwL>nko{tslm+{|r{zu2?MKgunkvgsHrE9sGVZ zV9)#Ft6!bjm+dRQz4^Z4w|{!W8^zS(SibjVvEp?@<)b*t~GU&C)7C`MOwa*M#{&09XBsuT80}f zw?IW$((9g6kN2;5UkT*Z`2Pz}&S2>F=brqvhE1{e8eKiVhGExWhz8IA8bAYR01co4 zH1Iz$5MxOn#{QQm=eJODmnkb5ldjO% z;VPv;E9EW5{5BegU8A&Yr&R8s^kl4MH0h-Iy^K#7ox5nhjPZ@@G>&Gx$7pwh=8G70 zyJ;NG*ulu*fduEeMGZhI1_mG<6=gJ!s>6XiM{lHqQAz0nGtW=M=||3)0N8)V}=^zE&iAS6<~ z{rtt<;OW&^sDZFk_NMaD%`zbzv0SaN6~q*&22)!?ajvE zdLhO@vay<^_NA`cXyr#7ZzW`hM@Y#1+NzepqU|IgGsLuJnv@ioRZN{57Ec@s9aQ%P z%Sh$20pEsk`$*-K#7`@Vnn|=)joTgfR06@@Yn2Nx6Aw-2C#^PzNPkNCuYGA(NsgN= zX7RYgB+pN5K3IK?JUuB}7TAzMX1Z@aVrAAz%GVsXIkoaI8LNNu+wSZxGCE@V4hbnF zS>E=2>G!*cmFQf?;|CSQ+O*4O@Ny?9Yc(4)*|46>_}L|KuC$#H<7W#t-IS8Jxu?ww zmtQ5^q+(gm_l@inUZpF44HIi*dW&StZ8%TF{#K&*d8P@VnD7(H8G60`e1!aExCBQNi}`qc_Pj(QkH&FMRd$_y#pM9jN7T?95~`f zQeJTBsaczRo&5rLH?to|#`_YN*vw3_Q`z{J+V(QCFw4Sj*5qW+Ip)=1+FMGzyn}0Y z#N>n2N$68_>J;hee>hXK;0&~C#QQ{bmXPU*et8cTRe*TnsO8%B#pJWu7Wp+-tKg_+ zRBYXhLehK4COupvfrzf}+7eb55W|PJ7kt>=0Cg9FLnQ0+2)_@JXaEhM0W^RH&;S}h n184vZpaC?12G9T+Km%w14WI!ufCkV28bAYR01cplKWpGOPr#@e literal 0 HcmV?d00001 diff --git a/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationE_uv.hdf5 b/examples/headland_inversion/outputs/outputs_forward/diagnostic_timeseries_stationE_uv.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..3d9d2331a63b7df43dcb1c6b414a4e246be20d45 GIT binary patch literal 23464 zcmeI2X;2hL6o6+zA|eW?L`pEODKrQkc*I-ZdY}kV5fKk40bvmgmjPTv4PixC42lPZ z5wwax$t4JK7>!WPsh}86QO+P~Z)ugU7mKB9?q zmbbDrG2!GZ`T9KBG-uq8j#T*s@`Ey9YO=sW@xbfYQwteOSf8jv`l|WvN|G)6$gngI4_2i}X#uR(6(YxpG;f*yIq5(932G9T+Km%w1 z4g60Ggo@~c9Hq>QWpNUvD2?SaC|k1G{5eWt4yA}t8s}5y6;N6fQMMFQdR?UCN+}Oq zq7;@>hC`%X*)YEuf z1EsK$vW+pliN=1-l%k(0`?XNIFqSbI-=O&j#vVr7R+`Ua)W1n%8RG*+i(53G#i(@x(cZ9Ew4gM;o$0mZ?eTT#CB+ZJrgVTV3ld4aO$gr?zkZhP>U-Tmy0N>|Ej9o$onPLw{nZQ74U%41+)IJO%Qn=;-SUPG-TuBa z7MurZc(vigr4qRBY?sm6RtUmzFIt*!`oodOXC_2+l|oL*-O&02U&D^|H|I}{E(e)f z{GA@XK-jyuZX`HX!SXXPmPyy7uuQLe`;5#QQ1-uldz&Z-atx22n3Puw1>+oV=?vZj z{=(YAxxv?ARPWqbtvx|t9zDs3JgtSMCbYC8)@2f4ba)?Gu111BJr24u|DqF0Mi?e z_|AOz6_JQ5pQn4*!?Ure%S#sQCL`2hoW)vo;G5d?GWMr1GE_}9DQfXGD5&4zS3CI- zd6C;O=wt1x&>LC(WZvb&WO)38OBW7T!B*ixE5pu%#85@z*_K@nrrm3dR*(>q+}dtX zr*#RQlsoI04cJKb_~!3S@+g9?fD{|8$UP8${Fbc#N-ju?4eQrhe+Tw9o9u-h8F0eX zATmlC3;Sv{o}U^P13{PdOtQ)np#4nqoca-=q(MEP$l}~dI8?jHzkggR(TT8@l`lL6 zPbKzN5{>i3=*+l4_5Eou(kg5FgxiIrb6-JG@r5*S|6Dtye#<3dt*h&irIrSz)+yf2 z+bW3u>0POgn^NJysJcgL5!FOy)HSH%VlsH1^j+KNT1yJ6jr$*(ngIMhM4|ySfCkV2 t8bAYR01co4G=K)s02)98XaEhM0W^RH&;S}h184vZpaC?12L7yp-vDjcrXT Date: Fri, 26 Jul 2024 13:52:00 +0100 Subject: [PATCH 06/15] Add DiagnosticCallback for inversion cost function - Also fix hanging in parallel when using weighting of stations in inversion examples --- examples/channel_inversion/inverse_problem.py | 4 +- examples/headland_inversion/Makefile | 11 ++- .../headland_inversion/inverse_problem.py | 29 ++++++- .../headland_inversion/inversion_tools_vel.py | 86 +++++++++++++------ examples/headland_inversion/model_config.py | 23 ++++- examples/tohoku_inversion/inverse_problem.py | 4 +- thetis/inversion_tools.py | 73 +++++++++++----- 7 files changed, 170 insertions(+), 60 deletions(-) diff --git a/examples/channel_inversion/inverse_problem.py b/examples/channel_inversion/inverse_problem.py index e480512eb..f19442fa0 100644 --- a/examples/channel_inversion/inverse_problem.py +++ b/examples/channel_inversion/inverse_problem.py @@ -111,9 +111,11 @@ # Extract the regularized cost function cost_function = inv_manager.get_cost_function(solver_obj) +cost_function_callback = inversion_tools.CostFunctionCallback(solver_obj, cost_function) +solver_obj.add_callback(cost_function_callback, 'timestep') # Solve and setup reduced functional -solver_obj.iterate(export_func=cost_function) +solver_obj.iterate() inv_manager.stop_annotating() # Run inversion diff --git a/examples/headland_inversion/Makefile b/examples/headland_inversion/Makefile index 1a5c6ad5a..883abd8a5 100644 --- a/examples/headland_inversion/Makefile +++ b/examples/headland_inversion/Makefile @@ -1,11 +1,18 @@ all: invert plot -CASE ?= IndependentPointsScheme # Constant Regions NodalFreedom +CASE ?= Constant # Regions IndependentPointsScheme NodalFreedom CONTROLS = Manning # This option is not used in this example - Manning only STATIONS = stationA stationB stationC stationD stationE +PARALLEL ?= 0 # Default to serial execution invert: - python3 inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test; \ +ifeq ($(PARALLEL), 0) + python3 inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test +else + for controls in $(CONTROLS); do \ + mpirun.mpich -np $(PARALLEL) python inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test; \ + done +endif plot: for station in $(STATIONS); do \ diff --git a/examples/headland_inversion/inverse_problem.py b/examples/headland_inversion/inverse_problem.py index cf8a798bf..b9d2e324b 100644 --- a/examples/headland_inversion/inverse_problem.py +++ b/examples/headland_inversion/inverse_problem.py @@ -6,6 +6,12 @@ from scipy.interpolate import LinearNDInterpolator, NearestNDInterpolator import h5py import argparse +from mpi4py import MPI + +# Initialize MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() # ---------------------------------------- Step 1: set up mesh and ground truth ---------------------------------------- @@ -55,8 +61,22 @@ coordinates = mesh2d.coordinates.dat.data[:] x, y = coordinates[:, 0], coordinates[:, 1] -lx, ly = np.max(x), np.max(y) -N = coordinates.shape[0] +local_lx = np.max(x) # for parallel runs, the mesh is partioned so we need to get the maximum from each processor! +local_ly = np.max(y) + +all_lx = comm.gather(local_lx, root=0) +all_ly = comm.gather(local_ly, root=0) +if rank == 0: + lx_ = np.max(all_lx) + ly_ = np.max(all_ly) +else: + lx_ = None + ly_ = None +lx = comm.bcast(lx_, root=0) +ly = comm.bcast(ly_, root=0) + +local_N = coordinates.shape[0] +N = comm.allreduce(local_N, op=MPI.SUM) # allreduce sums the local numbers to get the total number of coordinates masks, M, m_true = None, 0, [] # Create a FunctionSpace on the mesh (corresponds to Manning) @@ -159,7 +179,6 @@ def update_n(n, m): # Create station manager observation_data_dir = output_dir_forward variable = 'uv' -lx, ly = np.max(x), np.max(y) stations = [ ('stationA', (lx/10, ly/2)), ('stationB', (lx/2, ly/2)), @@ -230,9 +249,11 @@ def update_n(n, m): # Extract the regularized cost function cost_function = inv_manager.get_cost_function(solver_obj, weight_by_variance=True) +cost_function_callback = inversion_tools.CostFunctionCallback(solver_obj, cost_function) +solver_obj.add_callback(cost_function_callback, 'timestep') # Solve and setup reduced functional -solver_obj.iterate(update_forcings=update_forcings, export_func=cost_function) +solver_obj.iterate(update_forcings=update_forcings) inv_manager.stop_annotating() # Run inversion diff --git a/examples/headland_inversion/inversion_tools_vel.py b/examples/headland_inversion/inversion_tools_vel.py index 9d61e6eb8..dbd464249 100644 --- a/examples/headland_inversion/inversion_tools_vel.py +++ b/examples/headland_inversion/inversion_tools_vel.py @@ -7,6 +7,7 @@ from thetis.log import print_output from thetis.diagnostics import HessianRecoverer2D from thetis.exporter import HDF5Exporter +from thetis.callback import DiagnosticCallback import abc import numpy import h5py @@ -33,6 +34,32 @@ def group_consecutive_elements(indices): return grouped_list +class CostFunctionCallback(DiagnosticCallback): + def __init__(self, solver_obj, cost_function, **kwargs): + # Disable logging and HDF5 export + kwargs.setdefault('append_to_log', False) + kwargs.setdefault('export_to_hdf5', False) + super().__init__(solver_obj, **kwargs) + self.cost_function = cost_function + + @property + def name(self): + return 'cost_function_callback' + + @property + def variable_names(self): + return ['cost_function'] + + def __call__(self): + # Evaluate the cost function + cost_value = self.cost_function() + return [cost_value] + + def message_str(self, cost_value): + # Return a string representation of the cost function value + return f"Cost function value: {cost_value}" + + class InversionManager(FrozenHasTraits): """ Class for handling inversion problems and stashing @@ -331,11 +358,12 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): if weight_by_variance: var = fd.Function(self.sta_manager.fs_points_0d) - for i, j in enumerate(self.sta_manager.local_station_index): - obs_speed = numpy.sqrt( - numpy.array(self.sta_manager.observation_u[j]) ** 2 - + numpy.array(self.sta_manager.observation_v[j]) ** 2) - var.dat.data[i] = numpy.var(obs_speed) + if len(var.dat.data[:]) > 0: + for i, j in enumerate(self.sta_manager.local_station_index): + obs_speed = numpy.sqrt( + numpy.array(self.sta_manager.observation_u[j]) ** 2 + + numpy.array(self.sta_manager.observation_v[j]) ** 2) + var.dat.data[i] = numpy.var(obs_speed) self.sta_manager.station_weight_0d.interpolate(1 / var) def cost_fn(): @@ -557,29 +585,31 @@ def construct_evaluator(self): self.station_interpolators_u = [] self.station_interpolators_v = [] self.local_station_index = [] - for i in range(self.fs_points_0d.dof_dset.size): - # loop over local DOFs and match coordinates to observations - # NOTE this must be done manually as VertexOnlyMesh reorders points - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - xy_diff = xy - numpy.array([x_mesh, y_mesh]) - xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) - j = numpy.argmin(xy_dist) - self.local_station_index.append(j) - - x, y = xy[j, :] - t = self.observation_time[j] - u = self.observation_u[j] - v = self.observation_v[j] - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - - msg = 'bad station location ' \ - f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' - assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg - # create temporal interpolator - ip_u = interp1d(t, u, **interp_kw) - ip_v = interp1d(t, v, **interp_kw) - self.station_interpolators_u.append(ip_u) - self.station_interpolators_v.append(ip_v) + + if len(mesh0d.coordinates.dat.data[:]) > 0: + for i in range(self.fs_points_0d.dof_dset.size): + # loop over local DOFs and match coordinates to observations + # NOTE this must be done manually as VertexOnlyMesh reorders points + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + xy_diff = xy - numpy.array([x_mesh, y_mesh]) + xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) + j = numpy.argmin(xy_dist) + self.local_station_index.append(j) + + x, y = xy[j, :] + t = self.observation_time[j] + u = self.observation_u[j] + v = self.observation_v[j] + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + + msg = 'bad station location ' \ + f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' + assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg + # create temporal interpolator + ip_u = interp1d(t, u, **interp_kw) + ip_v = interp1d(t, v, **interp_kw) + self.station_interpolators_u.append(ip_u) + self.station_interpolators_v.append(ip_v) # Process start and end times for observations self.obs_start_times = numpy.array([ diff --git a/examples/headland_inversion/model_config.py b/examples/headland_inversion/model_config.py index f67197d9c..c0ed393d9 100644 --- a/examples/headland_inversion/model_config.py +++ b/examples/headland_inversion/model_config.py @@ -1,5 +1,11 @@ from thetis import * -from firedrake.output.vtk_output import VTKFile +from firedrake import VTKFile +from mpi4py import MPI + +# Initialize MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() def generate_bathymetry(mesh, H, output_directory, exporting=True): @@ -100,7 +106,20 @@ def construct_solver(store_station_time_series=True, **model_options): bathymetry_2d.project(bathy) x, y = SpatialCoordinate(mesh2d) - lx, ly = np.max(mesh2d.coordinates.dat.data[:, 0]), np.max(mesh2d.coordinates.dat.data[:, 1]) + local_lx = np.max(mesh2d.coordinates.dat.data[:, + 0]) # for parallel runs, the mesh is partioned so we need to get the maximum from each processor! + local_ly = np.max(mesh2d.coordinates.dat.data[:, 1]) + + all_lx = comm.gather(local_lx, root=0) + all_ly = comm.gather(local_ly, root=0) + if rank == 0: + lx_ = np.max(all_lx) + ly_ = np.max(all_ly) + else: + lx_ = None + ly_ = None + lx = comm.bcast(lx_, root=0) + ly = comm.bcast(ly_, root=0) # friction Manning coefficient manning_2d = Function(P1_2d, name='Manning coefficient') diff --git a/examples/tohoku_inversion/inverse_problem.py b/examples/tohoku_inversion/inverse_problem.py index c72858e8c..3f53f92e4 100644 --- a/examples/tohoku_inversion/inverse_problem.py +++ b/examples/tohoku_inversion/inverse_problem.py @@ -99,9 +99,11 @@ # Extract the regularized cost function cost_function = inv_manager.get_cost_function(solver_obj, weight_by_variance=True) +cost_function_callback = inversion_tools.CostFunctionCallback(solver_obj, cost_function) +solver_obj.add_callback(cost_function_callback, 'timestep') # Solve and setup the reduced functional -solver_obj.iterate(export_func=cost_function) +solver_obj.iterate() inv_manager.stop_annotating() # Run inversion diff --git a/thetis/inversion_tools.py b/thetis/inversion_tools.py index 232f0c5e2..a9d8bd214 100644 --- a/thetis/inversion_tools.py +++ b/thetis/inversion_tools.py @@ -7,6 +7,7 @@ from .log import print_output from .diagnostics import HessianRecoverer2D from .exporter import HDF5Exporter +from .callback import DiagnosticCallback import abc import numpy import h5py @@ -15,6 +16,32 @@ import os +class CostFunctionCallback(DiagnosticCallback): + def __init__(self, solver_obj, cost_function, **kwargs): + # Disable logging and HDF5 export + kwargs.setdefault('append_to_log', False) + kwargs.setdefault('export_to_hdf5', False) + super().__init__(solver_obj, **kwargs) + self.cost_function = cost_function + + @property + def name(self): + return 'cost_function_callback' + + @property + def variable_names(self): + return ['cost_function'] + + def __call__(self): + # Evaluate the cost function + cost_value = self.cost_function() + return [cost_value] + + def message_str(self, cost_value): + # Return a string representation of the cost function value + return f"Cost function value: {cost_value}" + + class InversionManager(FrozenHasTraits): """ Class for handling inversion problems and stashing @@ -248,8 +275,9 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): if weight_by_variance: var = fd.Function(self.sta_manager.fs_points_0d) - for i, j in enumerate(self.sta_manager.local_station_index): - var.dat.data[i] = numpy.var(self.sta_manager.observation_values[j]) + if len(var.dat.data[:]) > 0: + for i, j in enumerate(self.sta_manager.local_station_index): + var.dat.data[i] = numpy.var(self.sta_manager.observation_values[j]) self.sta_manager.station_weight_0d.interpolate(1/var) def cost_fn(): @@ -488,26 +516,27 @@ def construct_evaluator(self): # Construct timeseries interpolator self.station_interpolators = [] self.local_station_index = [] - for i in range(self.fs_points_0d.dof_dset.size): - # loop over local DOFs and match coordinates to observations - # NOTE this must be done manually as VertexOnlyMesh reorders points - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - xy_diff = xy - numpy.array([x_mesh, y_mesh]) - xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) - j = numpy.argmin(xy_dist) - self.local_station_index.append(j) - - x, y = xy[j, :] - t = self.observation_time[j] - v = self.observation_values[j] - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - - msg = 'bad station location ' \ - f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' - assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg - # create temporal interpolator - ip = interp1d(t, v, **interp_kw) - self.station_interpolators.append(ip) + if len(mesh0d.coordinates.dat.data[:]) > 0: + for i in range(self.fs_points_0d.dof_dset.size): + # loop over local DOFs and match coordinates to observations + # NOTE this must be done manually as VertexOnlyMesh reorders points + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + xy_diff = xy - numpy.array([x_mesh, y_mesh]) + xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) + j = numpy.argmin(xy_dist) + self.local_station_index.append(j) + + x, y = xy[j, :] + t = self.observation_time[j] + v = self.observation_values[j] + x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] + + msg = 'bad station location ' \ + f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' + assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg + # create temporal interpolator + ip = interp1d(t, v, **interp_kw) + self.station_interpolators.append(ip) # Process start and end times for observations self.obs_start_times = numpy.array([ From 719a1c27894a2528556aedbac64217ea00f54744 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Mon, 29 Jul 2024 10:22:43 +0100 Subject: [PATCH 07/15] Update README --- examples/headland_inversion/README.md | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/examples/headland_inversion/README.md b/examples/headland_inversion/README.md index 0f2a1e6bb..4374aa24f 100644 --- a/examples/headland_inversion/README.md +++ b/examples/headland_inversion/README.md @@ -9,9 +9,9 @@ configures the `solver object`, but with a friction field based on a sea bed par ![Sea Bed Particle Sizes](images/seabed_classification.png) -The idealised headland is 20km long and 6km wide, with a coastline depth of 3m and a main channel depth of 40m. Both -boundaries are undefined and are thus land boundaries. The boundaries are forced by a sinusoidal elevation function, -emulating a single tidal signal. A viscosity sponge is used at the left hand boundary to provide some model stability. +The idealised headland is 20km long and 6km wide, with a coastline depth of 3m and a main channel depth of 40m. The left +and right boundaries are forced by a sinusoidal elevation function, emulating a single tidal signal. A viscosity sponge +is used at the left hand boundary to provide some model stability. ## Inversion run @@ -22,8 +22,8 @@ source ~/firedrake/bin/activate make invert CASE=NodalFreedom ``` -The inversion problem is currently run from a `Makefile`. User arguments are specified here i.e. fields to optimise and -then the Makefile runs the scripts with the inputs provided. +The inversion problem is currently run from a `Makefile`. User arguments are specified here i.e. fields to optimise +(just Manning for this example) and then the Makefile runs the scripts with the inputs provided. The solver object is set up using `construct_solver` and then initial values for each field (in this case we only optimise for bed friction) are specified. The station manager, `StationObservationManager` , is then defined, which is @@ -44,13 +44,13 @@ friction is allowed, whilst in regions of lower resolution less variability is a data points. The cost function is then defined using the inversion manager, which is the Hessian regularised L2 norm. The actual class, `HessianRecoverer2D`, for calculating this loss can be found in `thetis.diagnostics.py`. -The forward model is then run (`solver_obj.iterate`), also passing `export_func=cost_function` to allow us to alter -the boundary conditions and perform an important step for the adjoint. Passing `export_func=cost_function` effectively -embeds the dependency of the model state on the control variables into the cost function, forming the reduced -functional. The reduced functional is thus the original cost function plus the regularization term, modified by the -model equations. Running the model gives us the baseline cost and more importantly, calculates the gradients of the cost -function with respect to the control variables (friction) which are fed into the adjoint method. The annotation process -is then stopped, which has been recording the computations related to the cost function and its derivatives. +The forward model is then run (`solver_obj.iterate`) with the cost function embedded via a callback, which is an +important step for the adjoint. Passing the cost function callback effectively embeds the dependency of the model state +on the control variables into the cost function, forming the reduced functional. The reduced functional is thus the +original cost function plus the regularization term, modified by the model equations. Running the model gives us the +baseline cost and more importantly, calculates the gradients of the cost function with respect to the control variables +(friction) which are fed into the adjoint method. The annotation process is then stopped, which has been recording the +computations related to the cost function and its derivatives. Optimisation parameters are then defined, which are the maximum number of iterations and the tolerance for the optimisation convergence criterion (threshold for the relative change in the cost function, below which the optimisation @@ -197,3 +197,14 @@ make plot CASE=IndependentPointsScheme To plot the progress, we can use the Makefile to run `plot_velocity_progress.py`. This plots the velocity over time at each of the station locations for each iteration of the optimisation, relative to the ground truth from the forward run. + +## Running in parallel + +The default settings run these scripts in parallel, however we can leverage parallel processing to accelerate the +simulations by partioning the mesh. To do so, simply provide the number of processors you would like to use after the +PARALLEL option, e.g.: + +```sh +source ~/firedrake/bin/activate +make invert CASE=IndependentPointsScheme PARALLEL=4 +``` From b875515153606dc0118c0fd76c032bf04f2b164c Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 13 Sep 2024 10:51:28 +0100 Subject: [PATCH 08/15] Update README - Add more introduction - Take the general inversion information out of the full nodal flexibility case - Add some references --- examples/headland_inversion/README.md | 88 ++++++++++++++++++--------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/examples/headland_inversion/README.md b/examples/headland_inversion/README.md index 4374aa24f..cd17c72e7 100644 --- a/examples/headland_inversion/README.md +++ b/examples/headland_inversion/README.md @@ -1,8 +1,29 @@ # Headland Channel - Friction Field Calibration +This example shows how adjoint methods can be used in gradient-based optimisation of a control parameter/field to +minimise the difference between measured and modelled data at gauging stations. Here, synthetic measured velocity data +is generated using a forward run with known input parameters. The calibration/optimisation problem is then performed +using the adjoint technique, keeping all the parameters the same except for our `Control`. 5 stations are used, as shown +in the forward run section. + +The control field used here is the Manning parameter, which controls the bottom drag i.e. the friction. This field can +take various 'mappings', which are covered case-by-case in the Inversion run sections: + +- Full nodal flexibility: the `Control` is the entire friction field, which is allowed to change at every single node on +the mesh, within the prescribed lower and upper limits. Hessian regularisation can be used to ensure node-to-node +changes are not too severe. See [Kärnä et al., 2023](https://doi.org/10.1029/2022MS003169) for detailed explanations. +- Uniform bed friction: the `Control` is one `Constant` value, which keeps the bed friction uniform across the domain. +- Region-based friction: the `Control` is a set of values, corresponding to certain regions of the model. These regions +may be a simple split between areas of the domain or may be based on seabed particle data - see +[Warder et al., 2022](http://dx.doi.org/https://doi.org/10.1007/s10236-022-01507-x), for example. +- Independent point scheme: as for the region-based approach, the `Control` is a set of values, but this time it is +associated with a set of points. Between points, the value of the friction parameter is interpolated - see +[Lu and Zhang, 2006](https://doi.org/10.1016/j.csr.2006.06.007), for example. + ## Forward run -The forward run is not included here, instead, only the time series `.hdf5` files at each station are provided. +The synthetic data is stored in the time series `.hdf5` files for each station. The forward run is provided so that the +input parameters can be changed by the user for further experimentation. The forward run used to generate this uses the same model configuration as provided by `model_config.py` which configures the `solver object`, but with a friction field based on a sea bed particle sizes as shown below: @@ -15,15 +36,8 @@ is used at the left hand boundary to provide some model stability. ## Inversion run -### Full Nodal Flexibility - -```sh -source ~/firedrake/bin/activate -make invert CASE=NodalFreedom -``` - -The inversion problem is currently run from a `Makefile`. User arguments are specified here i.e. fields to optimise -(just Manning for this example) and then the Makefile runs the scripts with the inputs provided. +The inversion problem is currently run from a `Makefile`. User arguments are specified here i.e. mapping to use, serial +or parallel execution, and then the Makefile runs the scripts with the inputs provided. The solver object is set up using `construct_solver` and then initial values for each field (in this case we only optimise for bed friction) are specified. The station manager, `StationObservationManager` , is then defined, which is @@ -34,23 +48,20 @@ optimised for, in this case, the velocity components. We can then set up the inversion manager, `InversionManager`, again from the modified version of the inversion tools of Thetis. The station manager is the first argument and the two key other arguments are the penalty parameters and cost -scaling. The cost function is regularised by an additional term, in this case, the Hessian (second derivative) of the -elevation field. This prevents overfitting of the Manning field i.e. having a highly variable field. The penalty -parameters are the regularisation parameters that control the strength of the regularisation. Higher values increase the -weight of the penalty term, leading to a smoother friction field, while lower values allow more variability. The cost -scaling normalises the regularisation term by the local mesh element size, so that the degree of penalization adapts to -the local mesh resolution. In regions with finer mesh resolution, the scaling ensures that higher variability in -friction is allowed, whilst in regions of lower resolution less variability is allowed to prevent overfitting to sparse -data points. The cost function is then defined using the inversion manager, which is the Hessian regularised L2 norm. -The actual class, `HessianRecoverer2D`, for calculating this loss can be found in `thetis.diagnostics.py`. - -The forward model is then run (`solver_obj.iterate`) with the cost function embedded via a callback, which is an -important step for the adjoint. Passing the cost function callback effectively embeds the dependency of the model state -on the control variables into the cost function, forming the reduced functional. The reduced functional is thus the -original cost function plus the regularization term, modified by the model equations. Running the model gives us the -baseline cost and more importantly, calculates the gradients of the cost function with respect to the control variables -(friction) which are fed into the adjoint method. The annotation process is then stopped, which has been recording the -computations related to the cost function and its derivatives. +scaling. We will use the penalty parameters only for the full nodal flexibility case, as the others are effectively +spatially regularised by their mappings. The cost function, J(u), is defined using the inversion manager as the L2 norm, +except in the full nodal flexibility case, where it has additional regularisation. + +The forward model is run using `solver_obj.iterate`, with the cost function embedded via a callback, which is an +important step for the adjoint method. Passing the cost function callback effectively embeds the dependency of the model +state on the control variables into the cost function, forming the **reduced functional**. The **reduced functional**, +J_hat(c), is not just the original cost function, J(u), plus a regularization term — it **includes the entire forward +model**. This means that to evaluate the reduced functional, all computations required to solve the model equations and +obtain the model state u(c) corresponding to a given control, c, are taken into account. Running the forward model not +only gives the baseline cost, but also calculates the gradients of the reduced functional with respect to the control +variables (e.g., friction), which are then passed to the adjoint method. Finally, the annotation process (which has been +recording all computations related to the cost function, its derivatives, and the forward model) is stopped. This +process allows us to efficiently compute the gradients needed for calibration. Optimisation parameters are then defined, which are the maximum number of iterations and the tolerance for the optimisation convergence criterion (threshold for the relative change in the cost function, below which the optimisation @@ -60,14 +71,33 @@ functional. These are the minimum and maximum values of bed friction allowed. The remainder of the script performs file saving and preparation for visualisation in ParaView. -### Constant Bed Friction +### Full Nodal Flexibility + +```sh +source ~/firedrake/bin/activate +make invert CASE=NodalFreedom +``` + +In the full nodal flexibility case, the friction values can vary freely within the lower and upper limits defined by the +control bounds. However, the cost function is regularised by an additional term, in this case, the Hessian (second +derivative) of the elevation field. This prevents overfitting of the Manning field i.e. having a highly variable field. +The penalty parameters are the regularisation parameters that control the strength of the regularisation. Higher values +increase the weight of the penalty term, leading to a smoother friction field, while lower values allow more +variability. The cost scaling normalises the regularisation term by the local mesh element size, so that the degree of +penalization adapts to the local mesh resolution. In regions with finer mesh resolution, the scaling ensures that higher +variability in friction is allowed, whilst in regions of lower resolution less variability is allowed to prevent +overfitting to sparse data points. The cost function that is defined using the inversion manager here, is the Hessian +regularised L2 norm. The actual class, `HessianRecoverer2D`, for calculating this loss can be found in +`thetis.diagnostics.py`. + +### Uniform Bed Friction ```sh source ~/firedrake/bin/activate make invert CASE=Constant ``` -For a constant bed friction, there are some differences which are enforced by changing the case entry, as explained +For a uniform bed friction, there are some differences which are enforced by changing the case entry, as explained below. Firstly, we do not need penalty parameters in the inversion problem as we will not have any variation across the field From a880ed75e9d0831f5e0c0462e55f96e5750a02cc Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 13 Sep 2024 10:54:08 +0100 Subject: [PATCH 09/15] Remove redundant options from the Makefile --- examples/headland_inversion/Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/headland_inversion/Makefile b/examples/headland_inversion/Makefile index 883abd8a5..6f8961c7d 100644 --- a/examples/headland_inversion/Makefile +++ b/examples/headland_inversion/Makefile @@ -1,7 +1,6 @@ all: invert plot CASE ?= Constant # Regions IndependentPointsScheme NodalFreedom -CONTROLS = Manning # This option is not used in this example - Manning only STATIONS = stationA stationB stationC stationD stationE PARALLEL ?= 0 # Default to serial execution @@ -9,9 +8,7 @@ invert: ifeq ($(PARALLEL), 0) python3 inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test else - for controls in $(CONTROLS); do \ - mpirun.mpich -np $(PARALLEL) python inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test; \ - done + mpiexec -np $(PARALLEL) python inverse_problem.py --case $(CASE) --no-consistency-test --no-taylor-test; endif plot: From f2f86b7949c6dcd0acb6c7aa8b65f36fa6a2caa7 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 13 Sep 2024 13:18:59 +0100 Subject: [PATCH 10/15] Update headland inversion mapping --- .../headland_inversion/inverse_problem.py | 92 +++++++------------ thetis/inversion_tools.py | 1 + 2 files changed, 35 insertions(+), 58 deletions(-) diff --git a/examples/headland_inversion/inverse_problem.py b/examples/headland_inversion/inverse_problem.py index b9d2e324b..fbc004cc7 100644 --- a/examples/headland_inversion/inverse_problem.py +++ b/examples/headland_inversion/inverse_problem.py @@ -87,18 +87,18 @@ manning_2d.assign(domain_constant(manning_const, mesh2d)) elif selected_case == 'Regions': # Define our values for n - mask_values = [ - np.logical_and(x < lx / 2, y < 1 * ly / 6).astype(float), - np.logical_and(x >= lx / 2, y < 1 * ly / 6).astype(float), - np.logical_and(np.logical_and(x < lx / 2, y >= 1 * ly / 6), y < 8 * ly / 15).astype(float), - np.logical_and(np.logical_and(x >= lx / 2, y >= 1 * ly / 6), y < 8 * ly / 15).astype(float), - np.logical_and(np.logical_and(x < 3 * lx / 8, y >= 8 * ly / 15), y < 5 * ly / 6).astype(float), - np.logical_and(np.logical_and(x >= 3 * lx / 8, x < 0.5 * lx), y >= 8 * ly / 15).astype(float), - np.logical_and(np.logical_and(x >= 0.5 * lx, x < 5 * lx / 8), y >= 8 * ly / 15).astype(float), - np.logical_and(np.logical_and(x >= 5 * lx / 8, y >= 8 * ly / 15), y < 5 * ly / 6).astype(float), - np.logical_and(x < 3 * lx / 8, y >= 5 * ly / 6).astype(float), - np.logical_and(x >= 5 * lx / 8, y >= 5 * ly / 6).astype(float) - ] + mask_values = np.array([ + ((x < lx / 2) & (y < ly / 6)), + ((x >= lx / 2) & (y < ly / 6)), + ((x < lx / 2) & (y >= ly / 6) & (y < 8 * ly / 15)), + ((x >= lx / 2) & (y >= ly / 6) & (y < 8 * ly / 15)), + ((x < 3 * lx / 8) & (y >= 8 * ly / 15) & (y < 5 * ly / 6)), + ((x >= 3 * lx / 8) & (x < 0.5 * lx) & (y >= 8 * ly / 15)), + ((x >= 0.5 * lx) & (x < 5 * lx / 8) & (y >= 8 * ly / 15)), + ((x >= 5 * lx / 8) & (y >= 8 * ly / 15) & (y < 5 * ly / 6)), + ((x < 3 * lx / 8) & (y >= 5 * ly / 6)), + ((x >= 5 * lx / 8) & (y >= 5 * ly / 6)) + ], dtype=float) m_true = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(len(mask_values))] masks = [Function(V) for _ in range(len(mask_values))] @@ -107,23 +107,19 @@ M = len(m_true) - # Define a function to update n based on m i.e. the mapping - def update_n(n, m): - # Reset n to zero - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, masks): - n += m_ * mask_ - - update_n(manning_2d, m_true) + manning_2d.assign(0) + for m_, mask_ in zip(m_true, masks): + manning_2d += m_ * mask_ elif selected_case == 'IndependentPointsScheme': # Define our values for n - points = [(0, 0), (0, 0.5 * ly), (0, ly), (0.5 * lx, 0), (lx, 0), (lx, 0.5 * ly), (lx, ly), (0.5 * lx, 0.5 * ly), - (0.1 * lx, 0.9 * ly), (0.3 * lx, 0.9 * ly), (0.7 * lx, 0.9 * ly), (0.9 * lx, 0.9 * ly), - (0.1 * lx, 0.7 * ly), (0.3 * lx, 0.7 * ly), (0.7 * lx, 0.7 * ly), (0.9 * lx, 0.7 * ly), - (0.3 * lx, 0.75 * ly), (0.7 * lx, 0.75 * ly), (0.5 * lx, 0.4 * ly), (0.5 * lx, 0.1 * ly), - (0.1 * lx, 0.25 * ly), (0.3 * lx, 0.25 * ly), (0.7 * lx, 0.25 * ly), (0.9 * lx, 0.25 * ly), - (0.1 * lx, 0.05 * ly), (0.3 * lx, 0.05 * ly), (0.7 * lx, 0.05 * ly), (0.9 * lx, 0.05 * ly)] + points = np.array([ + [0, 0], [0, 0.5], [0, 1], [0.5, 0], [1, 0], [1, 0.5], [1, 1], [0.5, 0.5], + [0.1, 0.9], [0.3, 0.9], [0.7, 0.9], [0.9, 0.9], + [0.1, 0.7], [0.3, 0.7], [0.7, 0.7], [0.9, 0.7], + [0.3, 0.75], [0.7, 0.75], [0.5, 0.4], [0.5, 0.1], + [0.1, 0.25], [0.3, 0.25], [0.7, 0.25], [0.9, 0.25], + [0.1, 0.05], [0.3, 0.05], [0.7, 0.05], [0.9, 0.05] + ]) * np.array([lx, ly]) m_true = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(len(points))] M = len(m_true) @@ -147,15 +143,9 @@ def update_n(n, m): for i, mask in enumerate(masks): mask.dat.data[:] = linear_coefficients[:, i] - # Define a function to update n based on m i.e. the mapping - def update_n(n, m): - # Reset n to zero - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, masks): - n += m_ * mask_ - - update_n(manning_2d, m_true) + manning_2d.assign(0) + for m_, mask_ in zip(m_true, masks): + manning_2d += m_ * mask_ else: pass @@ -226,16 +216,9 @@ def update_n(n, m): inv_manager.add_control(manning_const) elif selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': m_values = [Constant(0.03 - 0.0005 * i, domain=mesh2d) for i in range(M)] - - # Define a function to update n based on m i.e. the mapping - def update_n(n, m): - # Reset n to zero - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, masks): - n += m_ * mask_ - - update_n(manning_2d, m_values) + manning_2d.assign(0) + for m_, mask_ in zip(m_values, masks): + manning_2d += m_ * mask_ for i, control in enumerate(m_values): if i == 0: inv_manager.add_control(control, masks[0], new_map=True) @@ -281,16 +264,9 @@ def update_n(n, m): elif selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': P1_2d = get_functionspace(mesh2d, 'CG', 1) manning_2d = Function(P1_2d, name='manning2d') - - # Define a function to update n based on m i.e. the mapping - def update_n(n, m): - # Reset n to zero - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, masks): - n += m_ * mask_ - - update_n(manning_2d, inv_manager.m_list) + manning_2d.assign(0) + for m_, mask_ in zip(inv_manager.m_list, masks): + manning_2d += m_ * mask_ VTKFile(f'{options.output_directory}/manning_optimised.pvd').write(manning_2d) else: name = cc.name() @@ -300,5 +276,5 @@ def update_n(n, m): VTKFile(f'{options.output_directory}/{name}_optimised.pvd').write(oc) if selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': - print("Optimised vector m:\n", - [np.round(control_opt_list[i].dat.data[0], 4) for i in range(len(control_opt_list))]) + print_output("Optimised vector m:\n" + + str([np.round(control_opt_list[i].dat.data[0], 4) for i in range(len(control_opt_list))])) diff --git a/thetis/inversion_tools.py b/thetis/inversion_tools.py index 7a161704d..c571d375d 100644 --- a/thetis/inversion_tools.py +++ b/thetis/inversion_tools.py @@ -275,6 +275,7 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): if weight_by_variance: var = fd.Function(self.sta_manager.fs_points_0d) + # in parallel access to .dat.data should be collective if len(var.dat.data[:]) > 0: for i, j in enumerate(self.sta_manager.local_station_index): var.dat.data[i] = numpy.var(self.sta_manager.observation_values[j]) From 446e588164d811d3dd45cb300e583010ae5e725a Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 13 Sep 2024 13:36:14 +0100 Subject: [PATCH 11/15] Add headland inversion forward run - Small README update also - Adds necessary shapefiles for forward run --- examples/headland_inversion/README.md | 21 ++-- examples/headland_inversion/forward_run.py | 108 ++++++++++++++++++ .../headland_inversion/inputs/bed_classes.cpg | 1 + .../headland_inversion/inputs/bed_classes.dbf | Bin 0 -> 287 bytes .../headland_inversion/inputs/bed_classes.prj | 1 + .../headland_inversion/inputs/bed_classes.shp | Bin 0 -> 3100 bytes .../headland_inversion/inputs/bed_classes.shx | Bin 0 -> 172 bytes 7 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 examples/headland_inversion/forward_run.py create mode 100644 examples/headland_inversion/inputs/bed_classes.cpg create mode 100644 examples/headland_inversion/inputs/bed_classes.dbf create mode 100644 examples/headland_inversion/inputs/bed_classes.prj create mode 100644 examples/headland_inversion/inputs/bed_classes.shp create mode 100644 examples/headland_inversion/inputs/bed_classes.shx diff --git a/examples/headland_inversion/README.md b/examples/headland_inversion/README.md index cd17c72e7..e1609922a 100644 --- a/examples/headland_inversion/README.md +++ b/examples/headland_inversion/README.md @@ -147,24 +147,21 @@ associated with each mask. This means we can define each mask by assigning its v `mask.dat.data[:] = mask_values[i]`. In the case of bed particle size mapping, this is important, because it would be challenging to have to define a series of `conditional` or other operators to define each area. Note that if we included this assignment of the masks at each iteration of the adjoint, it would not work as there is no graphical -connection when we do assignments with `function.dat.data[:] = values`. We can then define our mapping function as -follows: +connection when we do assignments with `function.dat.data[:] = values`. We can then map our values as follows: ``` -def update_n(n, m): - # Reset n to zero - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, masks): - n += m_ * mask_ +manning_2d.assign(0) +# Add the weighted masks to n +for m_, mask_ in zip(m_true, masks): + manning_2d += m_ * mask_ ``` We then iterate through each mask (region) and then add the corresponding value of Manning (n) friction. We now have a mapping that `PyAdjoint` can understand. -As for the `InversionManager`, we can then provide this mapping `update_n` function which allows us to export the -`Control` and `Gradient` fields correctly, rather than having `m` outputs for `m` controls. We can then run the forward, -inverse and plotting scripts in order. +As for the `InversionManager`, we can then provide this mapping through an `update_n` function which allows us to export +the `Control` and `Gradient` fields correctly, rather than having `m` outputs for `m` controls. We can then run the +forward, inverse and plotting scripts in order. ### Independent Point Scheme @@ -180,7 +177,7 @@ interpolation of the points, this mapping is generated as follows: ``` # Get domain limits, define independent points and specify their values -lx, ly = np.max(x), np.max(y) +lx, ly = np.max(x), np.max(y) # done differently in parallel points = [(lx/4, ly/4), (lx/4, 3*ly/4), (lx/2, ly/4), (lx/2, 3*ly/4), (3*lx/4, ly/4), (3*lx/4, 3*ly/4)] m_true = [Constant(0.01*(i+1), domain=mesh2d) for i in range(len(points))] diff --git a/examples/headland_inversion/forward_run.py b/examples/headland_inversion/forward_run.py new file mode 100644 index 000000000..abd517400 --- /dev/null +++ b/examples/headland_inversion/forward_run.py @@ -0,0 +1,108 @@ +from thetis import * +from firedrake import * +from firedrake import VTKFile +import geopandas as gpd +from model_config import construct_solver +from shapely.geometry import Point +from mpi4py import MPI + +# Initialize MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + + +# ---------------------------------------- Step 1: set up mesh and ground truth ---------------------------------------- + +pwd = os.path.abspath(os.path.dirname(__file__)) +output_dir_forward = f'{pwd}/outputs/outputs_forward' + +solver_obj, update_forcings = construct_solver( + output_directory=output_dir_forward, + store_station_time_series=True, + no_exports=False, +) + +mesh2d = solver_obj.mesh2d +options = solver_obj.options +manning_2d = solver_obj.fields.manning_2d +elev_init_2d = solver_obj.fields.elev_2d + +coordinates = mesh2d.coordinates.dat.data[:] +x, y = coordinates[:, 0], coordinates[:, 1] +local_lx = np.max(x) # for parallel runs, the mesh is partioned so we need to get the maximum from each processor! +local_ly = np.max(y) + +all_lx = comm.gather(local_lx, root=0) +all_ly = comm.gather(local_ly, root=0) +if rank == 0: + lx_ = np.max(all_lx) + ly_ = np.max(all_ly) +else: + lx_ = None + ly_ = None +lx = comm.bcast(lx_, root=0) +ly = comm.bcast(ly_, root=0) + +# Create a FunctionSpace on the mesh (corresponds to Manning) +V = get_functionspace(mesh2d, 'CG', 1) + +# Load the shapefile +shapefile_path = 'inputs/bed_classes.shp' +gdf = gpd.read_file(shapefile_path) +polygons_by_id = gdf.groupby('id') + +sediment_to_manning = { + 'ROCK': 0.0420, + 'SAND': 0.0171, + 'SANDY CLAY': 0.0132, + 'MUDDY SAND': 0.0163, + 'CLAY': 0.0100 +} + +mask_values = [] +masks = [Function(V) for _ in range(len(polygons_by_id))] +m_true = [] + +for i, (region_id, group) in enumerate(polygons_by_id): + multi_polygon = group.unary_union + + # Get the sediment type for this region (assuming one sediment type per ID) + sediment_type = group['Sediment'].iloc[0] + manning_value = sediment_to_manning.get(sediment_type, None) + values = [] + + for (x_, y_) in zip(x, y): + # Check if the point is inside the multi-polygon + point = Point(x_, y_) + if multi_polygon.buffer(1).contains(point): + values.append(1) + else: + values.append(0) + + mask_values.append(values) + m_true.append(Constant(manning_value, domain=mesh2d)) + +overlap_counts = np.zeros(len(x)) + +for values in mask_values: + overlap_counts += np.array(values) + +for values in mask_values: + for i in range(len(values)): + if overlap_counts[i] > 1: + values[i] /= overlap_counts[i] + +for mask, values in zip(masks, mask_values): + mask.dat.data[:] = values + +manning_2d.assign(0) +for m_, mask_ in zip(m_true, masks): + manning_2d += m_ * mask_ + +VTKFile(output_dir_forward + '/manning_init.pvd').write(manning_2d) + +print_output('Exporting to ' + solver_obj.options.output_directory) + +print_output('Solving the forward problem...') +solver_obj.iterate(update_forcings=update_forcings) diff --git a/examples/headland_inversion/inputs/bed_classes.cpg b/examples/headland_inversion/inputs/bed_classes.cpg new file mode 100644 index 000000000..3ad133c04 --- /dev/null +++ b/examples/headland_inversion/inputs/bed_classes.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/examples/headland_inversion/inputs/bed_classes.dbf b/examples/headland_inversion/inputs/bed_classes.dbf new file mode 100644 index 0000000000000000000000000000000000000000..f80e9ff80a3533ca1ed0cab42dd6f0f6f009fdf9 GIT binary patch literal 287 zcmZRsVdvyzU|>jO5CxK$ATtFn<_BVN!MPAdaB50sZfaf$kmn2%L6_%MfCEF{P#2d- zgQ5=VHsmnX@bp6Q*7GJK-wb}P^HYVX}1XSclL(ZDFpz& CMkl2J literal 0 HcmV?d00001 diff --git a/examples/headland_inversion/inputs/bed_classes.prj b/examples/headland_inversion/inputs/bed_classes.prj new file mode 100644 index 000000000..9855fcc1f --- /dev/null +++ b/examples/headland_inversion/inputs/bed_classes.prj @@ -0,0 +1 @@ +PROJCS["WGS_1984_UTM_Zone_30N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-3.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/examples/headland_inversion/inputs/bed_classes.shp b/examples/headland_inversion/inputs/bed_classes.shp new file mode 100644 index 0000000000000000000000000000000000000000..f055cc0f01df0fdb3b10bb4b3428be31ddf84203 GIT binary patch literal 3100 zcmZ`*3s6*L6#fN*FhR80$Qm`WhSyls^l}8|P)#F(4`f0EbCL+f2aB*eniPf>IG|!Y zW(?7299s|;To!Of!QIO~0-6wcmRMj(LI|Oh7>;!J|L?zCj?107_uTLP=bZnX^Pk5h z#CtZedpWz`awOz2yKCad%r>A)NwthpFymtuA+wUKx-R#%CG6>}huF6}k_YydfTl>% zY&>HD-KL@bua&hRG7^7D4d>-+bMuQdSpj-TF!qHU+*%7^j#1Y&cl6+r_}qAfZ!LV} zb;PnV`Uh?H`i)PUq@W6C^EV_H|~IJRh;p z4B5{z`v>oCj!^w(hUN>*KOUQ*4_Rz(b)*}794&D2bZTx%s1DA~ON|M> zI(RE~a9#XKGmI$JOa6A#!ONxF_s`#IhVfSuZWOAjz+qAL(_>y{2-e&jaClz}4d=CA zEbcTxU|)1h>VK8s;k;;mPKyatkxP?Yf31WT2jzj}eI{t%(U`6HR|5w-!sh()f(f4T z+`XgRyBt!4jUJ(eM!2$}y-B<99P~yDvq#=ELgV=M&V1KWSX|lf|CzDDcGjmeJ=@d6 z8sOmrcJ>YTm;d9@EkL$sBIVy{DITd&37{zLqWWkW_zCcOY+A`H%gR9IP<8jkzYGu+ z|0&)7J#|jnkpcr`v^7)vnyUJ4E5QI|SuauhqZF$??_FkqZ*Rm<|F&pS{(NFs5B_=m z)c*qCyhFFY*F)D;7N60t6y+yU^zg?$7|O|>mkyI#rMme)Znf;dU#FE;*axF zwAVqr*UD>#^Kx10lUOs>Ja?gB>apqKyZgvm9ZY=@-$Y)#57)(Y5AF^L{QakT2z82Y z&0C`bFClJC{;E2-cfx&qLd0C3Outy!WQEmEoKuunR@+uPo$1F%d2xPXS&qAXw0p)x zJG}Q77sjY9koF*mn950CA7gzuZC^RJZf21(zfbT(YSEdGR$$HjZ<97RAczL(f*XFNw)AEq+%1fZ1f zV>U3%?eLT^_bd4_TjzWA)KAC57a8XK+b%H7^`}-d%;OkW!Z43>KrzG8IS@F;Fh3_E zUHlwH$bF?NdnJ3sJiz@G&jCCq@EpN&2G1cpr|=xZbB@|KJ^rrZ|5Sjj9wpa^*azsV zEN89txWu+DzR2|8d5h;Up4WJu<9Uz&2k5rb^N$dmpE!SUe&hVd^?~aL*B7e4N9q&R xFTwRKizW4kV;*Z;Y^B(9%<~NAA Date: Fri, 13 Sep 2024 14:36:38 +0100 Subject: [PATCH 12/15] Exclude headland_inversion forward run from testing - So that we don't need geopandas as a dependency - The forward run is tested during the inversion run anyway - Also fix a lint issue in inversion script --- examples/headland_inversion/inverse_problem.py | 3 +-- test/examples/test_examples.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/headland_inversion/inverse_problem.py b/examples/headland_inversion/inverse_problem.py index fbc004cc7..e73928120 100644 --- a/examples/headland_inversion/inverse_problem.py +++ b/examples/headland_inversion/inverse_problem.py @@ -276,5 +276,4 @@ VTKFile(f'{options.output_directory}/{name}_optimised.pvd').write(oc) if selected_case == 'Regions' or selected_case == 'IndependentPointsScheme': - print_output("Optimised vector m:\n" + - str([np.round(control_opt_list[i].dat.data[0], 4) for i in range(len(control_opt_list))])) + print_output("Optimised vector m:\n" + str([np.round(control_opt_list[i].dat.data[0], 4) for i in range(len(control_opt_list))])) diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index d4790a7f6..8ab885cc4 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -39,6 +39,7 @@ 'tidalfarm/tidalfarm.py', 'tidal_barrage/plotting.py', 'channel_inversion/plot_elevation_progress.py', + 'headland_inversion/forward_run.py', 'headland_inversion/inversion_tools_vel.py', 'headland_inversion/plot_velocity_progress.py', 'tohoku_inversion/okada.py', From c2d32e9eb78c9d4c09fc574dd7833d6d5cc7dfbd Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Fri, 13 Sep 2024 14:49:35 +0100 Subject: [PATCH 13/15] Remove duplicate testing --- test/examples/test_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index 8ab885cc4..894581541 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -39,7 +39,9 @@ 'tidalfarm/tidalfarm.py', 'tidal_barrage/plotting.py', 'channel_inversion/plot_elevation_progress.py', + 'channel_inversion/inverse_problem.py', 'headland_inversion/forward_run.py', + 'headland_inversion/inverse_problem.py', 'headland_inversion/inversion_tools_vel.py', 'headland_inversion/plot_velocity_progress.py', 'tohoku_inversion/okada.py', From ce23e8e8ea9aeaeff7271f0e7013e514bf861a33 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Mon, 16 Sep 2024 20:25:25 +0100 Subject: [PATCH 14/15] Provide uv as option for calibration in inversion tools - Update examples accordingly --- examples/channel_inversion/inverse_problem.py | 2 +- .../headland_inversion/inverse_problem.py | 10 +- .../headland_inversion/inversion_tools_vel.py | 827 ------------------ examples/tohoku_inversion/inverse_problem.py | 2 +- thetis/inversion_tools.py | 362 ++++++-- 5 files changed, 292 insertions(+), 911 deletions(-) delete mode 100644 examples/headland_inversion/inversion_tools_vel.py diff --git a/examples/channel_inversion/inverse_problem.py b/examples/channel_inversion/inverse_problem.py index dfc833cf3..4b42d05e9 100644 --- a/examples/channel_inversion/inverse_problem.py +++ b/examples/channel_inversion/inverse_problem.py @@ -90,7 +90,7 @@ ] sta_manager = inversion_tools.StationObservationManager( mesh2d, output_directory=options.output_directory) -sta_manager.load_observation_data(observation_data_dir, station_names, variable) +sta_manager.load_elev_observation_data(observation_data_dir, station_names, variable) sta_manager.set_model_field(solver_obj.fields.elev_2d) # Define the scaling for the cost function so that J ~ O(1) diff --git a/examples/headland_inversion/inverse_problem.py b/examples/headland_inversion/inverse_problem.py index e73928120..4d1222ba9 100644 --- a/examples/headland_inversion/inverse_problem.py +++ b/examples/headland_inversion/inverse_problem.py @@ -1,5 +1,5 @@ from thetis import * -import inversion_tools_vel as inversion_tools +import thetis.inversion_tools as inversion_tools from firedrake import * from firedrake.adjoint import * from model_config import construct_solver @@ -176,7 +176,8 @@ ('stationD', (3*lx/4, 3*ly/4)), ('stationE', (9*lx/10, ly/2)), ] -sta_manager = inversion_tools.StationObservationManager(mesh2d, output_directory=options.output_directory) +sta_manager = inversion_tools.StationObservationManager(mesh2d, output_directory=options.output_directory, + observation_param='uv') print_output('Station Manager instantiated.') station_names, observation_coords, observation_time, observation_u, observation_v = [], [], [], [], [] for name, (sta_x, sta_y) in stations: @@ -190,9 +191,8 @@ observation_u.append(var[:, 0]) observation_v.append(var[:, 1]) observation_x, observation_y = numpy.array(observation_coords).T -sta_manager.register_observation_data(station_names, variable, observation_time, observation_u, - observation_v, observation_x, observation_y, - start_times=None, end_times=None) +sta_manager.register_observation_data(station_names, variable, observation_time, observation_x, observation_y, + u=observation_u, v=observation_v, start_times=None, end_times=None) print_output('Data registered.') sta_manager.construct_evaluator() sta_manager.set_model_field(solver_obj.fields.uv_2d) diff --git a/examples/headland_inversion/inversion_tools_vel.py b/examples/headland_inversion/inversion_tools_vel.py deleted file mode 100644 index dbd464249..000000000 --- a/examples/headland_inversion/inversion_tools_vel.py +++ /dev/null @@ -1,827 +0,0 @@ -import firedrake as fd -from firedrake.adjoint import * -import ufl -from thetis.configuration import FrozenHasTraits -from thetis.solver2d import FlowSolver2d -from thetis.utility import create_directory, print_function_value_range, get_functionspace, unfrozen, domain_constant -from thetis.log import print_output -from thetis.diagnostics import HessianRecoverer2D -from thetis.exporter import HDF5Exporter -from thetis.callback import DiagnosticCallback -import abc -import numpy -import h5py -from scipy.interpolate import interp1d -import time as time_mod -import os - - -def group_consecutive_elements(indices): - grouped_list = [] - current_group = [] - - for index in indices: - if not current_group or index == current_group[-1]: - current_group.append(index) - else: - grouped_list.append(current_group) - current_group = [index] - - # Append the last group - if current_group: - grouped_list.append(current_group) - - return grouped_list - - -class CostFunctionCallback(DiagnosticCallback): - def __init__(self, solver_obj, cost_function, **kwargs): - # Disable logging and HDF5 export - kwargs.setdefault('append_to_log', False) - kwargs.setdefault('export_to_hdf5', False) - super().__init__(solver_obj, **kwargs) - self.cost_function = cost_function - - @property - def name(self): - return 'cost_function_callback' - - @property - def variable_names(self): - return ['cost_function'] - - def __call__(self): - # Evaluate the cost function - cost_value = self.cost_function() - return [cost_value] - - def message_str(self, cost_value): - # Return a string representation of the cost function value - return f"Cost function value: {cost_value}" - - -class InversionManager(FrozenHasTraits): - """ - Class for handling inversion problems and stashing - the progress of the associated optimization routines. - """ - - @unfrozen - def __init__(self, sta_manager, output_dir='outputs', no_exports=False, real=False, - penalty_parameters=[], cost_function_scaling=None, - test_consistency=True, test_gradient=True): - """ - :arg sta_manager: the :class:`StationManager` instance - :kwarg output_dir: model output directory - :kwarg no_exports: if True, nothing will be written to disk - :kwarg real: is the inversion in the Real space? - :kwarg penalty_parameters: a list of penalty parameters to pass - to the :class:`ControlRegularizationManager` - :kwarg cost_function_scaling: global scaling for the cost function. - As rule of thumb, it's good to scale the functional to J < 1. - :kwarg test_consistency: toggle testing the correctness with - which the :class:`ReducedFunctional` can recompute values - :kwarg test_gradient: toggle testing the correctness with - which the :class:`ReducedFunctional` can recompute gradients - """ - assert isinstance(sta_manager, StationObservationManager) - self.sta_manager = sta_manager - self.reg_manager = None - self.output_dir = output_dir - self.no_exports = no_exports or real - self.real = real - self.maps = [] - self.penalty_parameters = penalty_parameters - self.cost_function_scaling = cost_function_scaling or fd.Constant(1.0) - self.sta_manager.cost_function_scaling = self.cost_function_scaling - self.test_consistency = test_consistency - self.test_gradient = test_gradient - self.outfiles_m = [] - self.outfiles_dJdm = [] - self.outfile_index = [] - self.control_exporters = [] - self.initialized = False - - self.J = 0 # cost function value (float) - self.J_reg = 0 # regularization term value (float) - self.J_misfit = 0 # misfit term value (float) - self.dJdm_list = None # cost function gradient (Function) - self.m_list = None # control (Function) - self.Jhat = None - self.m_progress = [] - self.J_progress = [] - self.J_reg_progress = [] - self.J_misfit_progress = [] - self.dJdm_progress = [] - self.i = 0 - self.tic = None - self.nb_grad_evals = 0 - self.control_coeff_list = [] - self.control_list = [] - self.control_family = [] - - def initialize(self): - if not self.no_exports: - if self.real: - raise ValueError("Exports are not supported in Real mode.") - create_directory(self.output_dir) - create_directory(self.output_dir + '/hdf5') - for i in range(max(self.outfile_index)+1): - self.outfiles_m.append( - fd.VTKFile(f'{self.output_dir}/control_progress_{i:02d}.pvd')) - self.outfiles_dJdm.append( - fd.VTKFile(f'{self.output_dir}/gradient_progress_{i:02d}.pvd')) - self.initialized = True - - def add_control(self, f, mapping=None, new_map=False): - """ - Add a control field. - - Can be called multiple times in case of multiparameter optimization. - - :arg f: Function or Constant to be used as a control variable. - :kwarg mapping: map that dictates how each Control relates to the function being altered - :kwarg new_map: if True, create a new exporter - """ - if len(self.control_coeff_list) == 0: - new_map = True - if mapping: - self.maps.append(mapping) - else: - self.maps.append(None) - self.control_coeff_list.append(f) - self.control_list.append(Control(f)) - function_space = f.function_space() - element = function_space.ufl_element() - family = element.family() - self.control_family.append(family) - if isinstance(f, fd.Function) and not self.no_exports: - j = len(self.control_coeff_list) - 1 - prefix = f'control_{j:02d}' - if family == 'Real': # i.e. Control is a Constant (assigned to a mesh, so won't register as one) - if new_map: - self.control_exporters.append( - HDF5Exporter(get_functionspace(self.sta_manager.mesh, 'CG', 1), self.output_dir + '/hdf5', - prefix) - ) - if len(self.control_coeff_list) == 1: - self.outfile_index.append(0) - else: - self.outfile_index.append(self.outfile_index[-1] + 1) - else: - self.outfile_index.append(self.outfile_index[-1]) - else: # i.e. Control is a Function - self.control_exporters.append( - HDF5Exporter(function_space, self.output_dir + '/hdf5', prefix) - ) - if len(self.control_coeff_list) == 1: - self.outfile_index.append(0) - else: - self.outfile_index.append(self.outfile_index[-1] + 1) - - def reset_counters(self): - self.nb_grad_evals = 0 - - def set_control_state(self, j, djdm_list, m_list): - """ - Stores optimization state. - - To call whenever variables are updated. - - :arg j: error functional value - :arg djdm_list: list of gradient functions - :arg m_list: list of control coefficents - """ - self.J = j - self.dJdm_list = djdm_list - self.m_list = m_list - - tape = get_working_tape() - reg_blocks = tape.get_blocks(tag="reg_eval") - self.J_reg = sum([b.get_outputs()[0].saved_output for b in reg_blocks]) - misfit_blocks = tape.get_blocks(tag="misfit_eval") - self.J_misfit = sum([b.get_outputs()[0].saved_output for b in misfit_blocks]) - - def start_clock(self): - self.tic = time_mod.perf_counter() - - def stop_clock(self): - toc = time_mod.perf_counter() - return toc - - def set_initial_state(self, *state): - self.set_control_state(*state) - self.update_progress() - - def update_progress(self): - """ - Updates optimization progress and stores variables to disk. - - To call after successful line searches. - """ - toc = self.stop_clock() - if self.i == 0: - for f in self.control_coeff_list: - print_function_value_range(f, name=f.name, prefix='Initial') - - elapsed = '-' if self.tic is None else f'{toc - self.tic:.1f} s' - self.tic = toc - - if not self.initialized: - self.initialize() - - # cost function and gradient norm output - djdm = [fd.norm(f) for f in self.dJdm_list] - if self.real: - controls = [m.dat.data[0] for m in self.m_list] - self.m_progress.append(controls) - self.J_progress.append(self.J) - self.J_reg_progress.append(self.J_reg) - self.J_misfit_progress.append(self.J_misfit) - self.dJdm_progress.append(djdm) - comm = self.control_coeff_list[0].comm - if comm.rank == 0 and not self.no_exports: - if self.real: - numpy.save(f'{self.output_dir}/m_progress', self.m_progress) - numpy.save(f'{self.output_dir}/J_progress', self.J_progress) - numpy.save(f'{self.output_dir}/J_reg_progress', self.J_reg_progress) - numpy.save(f'{self.output_dir}/J_misfit_progress', self.J_misfit_progress) - numpy.save(f'{self.output_dir}/dJdm_progress', self.dJdm_progress) - if len(djdm) > 10: - djdm = f"[{numpy.min(djdm):.4e} .. {numpy.max(djdm):.4e}]" - else: - djdm = "[" + ", ".join([f"{dj:.4e}" for dj in djdm]) + "]" - print_output(f'line search {self.i:2d}: ' - f'J={self.J:.3e}, dJdm={djdm}, ' - f'grad_ev={self.nb_grad_evals}, duration {elapsed}') - - if not self.no_exports: - grouped_indices = group_consecutive_elements(self.outfile_index) - ref_index = 0 - for j in range(len(self.outfiles_m)): - m = self.m_list[ref_index] - o_m = self.outfiles_m[j] - o_jm = self.outfiles_dJdm[j] - self.dJdm_list[ref_index].rename('Gradient') - e = self.control_exporters[j] - if len(grouped_indices[j]) > 1: - # can only be Constants - mesh2d = self.sta_manager.mesh - P1_2d = get_functionspace(mesh2d, 'CG', 1) - # vtk format - field = fd.Function(P1_2d, name='control') - self.update_n(field, self.m_list[ref_index:ref_index + len(grouped_indices[j])], - ref_index, ref_index + len(grouped_indices[j])) - o_m.write(field) - # hdf5 format - e.export(field) - # gradient output - gradient = fd.Function(P1_2d, name='Gradient') - self.update_n(gradient, self.dJdm_list[ref_index:ref_index + len(grouped_indices[j])], - ref_index, ref_index + len(grouped_indices[j])) - o_jm.write(gradient) - else: - if self.control_family[ref_index] == 'Real': - mesh2d = self.sta_manager.mesh - P1_2d = get_functionspace(mesh2d, 'CG', 1) - # vtk format - field = fd.Function(P1_2d, name='control') - field.assign(domain_constant(m, mesh2d)) - o_m.write(field) - # hdf5 format - e.export(field) - # gradient output - gradient = fd.Function(P1_2d, name='Gradient') - gradient.assign(domain_constant(self.dJdm_list[ref_index], mesh2d)) - o_jm.write(gradient) - else: - # control output - # vtk format - m.rename(self.control_coeff_list[j].name()) - o_m.write(m) - # hdf5 format - e.export(m) - # gradient output - o_jm.write(self.dJdm_list[ref_index]) - ref_index += len(grouped_indices[j]) - - self.i += 1 - self.reset_counters() - - @property - def rf_kwargs(self): - """ - Default keyword arguments to pass to the - :class:`ReducedFunctional` class. - """ - def gradient_eval_cb(j, djdm, m): - self.set_control_state(j, djdm, m) - self.nb_grad_evals += 1 - return djdm - - params = { - 'derivative_cb_post': gradient_eval_cb, - } - return params - - def get_cost_function(self, solver_obj, weight_by_variance=False): - r""" - Get a sum of square errors cost function for the problem: - - ..math:: - J(u) = \sum_{i=1}^{n_{ts}} \sum_{j=1}^{n_{sta}} (u_j^{(i)} - u_{j,o}^{(i)})^2, - - where :math:`u_{j,o}^{(i)}` and :math:`u_j^{(i)}` denote the - observed and computed values at timestep :math:`i`, and - :math:`n_{ts}` and :math:`n_{sta}` are the numbers of timesteps - and stations, respectively. - - Regularization terms are included if a - :class:`RegularizationManager` instance is provided. - - :arg solver_obj: the :class:`FlowSolver2d` instance - :kwarg weight_by_variance: should the observation data be - weighted by the variance at each station? - """ - assert isinstance(solver_obj, FlowSolver2d) - if len(self.penalty_parameters) > 0: - self.reg_manager = ControlRegularizationManager( - self.control_coeff_list, - self.penalty_parameters, - self.cost_function_scaling, - RSpaceRegularizationCalculator if self.real else HessianRegularizationCalculator) - self.J_reg = 0 - self.J_misfit = 0 - if self.reg_manager is not None: - self.J_reg = self.reg_manager.eval_cost_function() - self.J = self.J_reg - - if weight_by_variance: - var = fd.Function(self.sta_manager.fs_points_0d) - if len(var.dat.data[:]) > 0: - for i, j in enumerate(self.sta_manager.local_station_index): - obs_speed = numpy.sqrt( - numpy.array(self.sta_manager.observation_u[j]) ** 2 - + numpy.array(self.sta_manager.observation_v[j]) ** 2) - var.dat.data[i] = numpy.var(obs_speed) - self.sta_manager.station_weight_0d.interpolate(1 / var) - - def cost_fn(): - t = solver_obj.simulation_time - misfit = self.sta_manager.eval_cost_function(t) - self.J_misfit += misfit - self.J += misfit - - return cost_fn - - @property - def reduced_functional(self): - """ - Create a Pyadjoint :class:`ReducedFunctional` for the optimization. - """ - if self.Jhat is None: - self.Jhat = ReducedFunctional(self.J, self.control_list, **self.rf_kwargs) - return self.Jhat - - def stop_annotating(self): - """ - Stop recording operations for the adjoint solver. - - This method should be called after the :meth:`iterate` - method of :class:`FlowSolver2d`. - """ - assert self.reduced_functional is not None - if self.test_consistency: - self.consistency_test() - if self.test_gradient: - self.taylor_test() - pause_annotation() - - def get_optimization_callback(self): - """ - Get a callback for stashing optimization progress - after successful line search. - """ - - def optimization_callback(m): - self.update_progress() - if not self.no_exports: - self.sta_manager.dump_time_series() - - return optimization_callback - - def update_n(self, n, m, start_index, end_index): - """ - Update the function `n` by adding weighted masks from `m`. - - This method resets `n` to zero and then adds the weighted masks defined in - `self.maps` to `n`. Each element in `m` is multiplied by the corresponding - mask before being added to `n`. - - :param n: Function or Field to be updated. - :param m: List of Constants/Functions to be combined with the masks. - :param start_index: Starting map. - :param end_index: Final map. - :raises ValueError: If the lengths of `m` and `self.maps` do not match. - """ - n.assign(0) - # Add the weighted masks to n - for m_, mask_ in zip(m, self.maps[start_index:end_index]): - n += m_ * mask_ - - def minimize(self, opt_method="BFGS", bounds=None, **opt_options): - """ - Minimize the reduced functional using a given optimization routine. - - :kwarg opt_method: the optimization routine - :kwarg bounds: a list of bounds to pass to the optimization routine - :kwarg opt_options: other optimization parameters to pass - """ - print_output(f'Running {opt_method} optimization') - self.reset_counters() - self.start_clock() - J = float(self.reduced_functional(self.control_coeff_list)) - self.set_initial_state(J, self.reduced_functional.derivative(), self.control_coeff_list) - if not self.no_exports: - self.sta_manager.dump_time_series() - return minimize( - self.reduced_functional, method=opt_method, bounds=bounds, - callback=self.get_optimization_callback(), options=opt_options) - - def consistency_test(self): - """ - Test that :attr:`reduced_functional` can correctly recompute the - objective value, assuming that none of the controls have changed - since it was created. - """ - print_output("Running consistency test") - J = self.reduced_functional(self.control_coeff_list) - if not numpy.isclose(J, self.J): - raise ValueError(f"Consistency test failed (expected {self.J}, got {J})") - print_output("Consistency test passed!") - - def taylor_test(self): - """ - Run a Taylor test to check that the :attr:`reduced_functional` can - correctly compute consistent gradients. - - Note that the Taylor test is applied on the current control values. - """ - func_list = [] - for f in self.control_coeff_list: - dc = f.copy(deepcopy=True) - func_list.append(dc) - minconv = taylor_test(self.reduced_functional, self.control_coeff_list, func_list) - if minconv < 1.9: - raise ValueError("Taylor test failed") # NOTE: Pyadjoint already prints the testing - print_output("Taylor test passed!") - - -class StationObservationManager: - """ - Implements error functional based on observation time series. - - The functional is the squared sum of error between the model and - observations. - - This object evaluates the model fields at the station locations, - interpolates the observations time series to the model time, computes the - error functional, and also stores the model's time series data to disk. - """ - def __init__(self, mesh, output_directory='outputs'): - """ - :arg mesh: the 2D mesh object. - :kwarg output_directory: directory where model time series are stored. - """ - self.mesh = mesh - on_sphere = self.mesh.geometric_dimension() == 3 - if on_sphere: - raise NotImplementedError('Sphere meshes are not supported yet.') - self.cost_function_scaling = fd.Constant(1.0) - self.output_directory = output_directory - # keep observation time series in memory - self.obs_func_u_list = [] - self.obs_func_v_list = [] - # keep model time series in memory during optimization progress - self.station_value_u_progress = [] - self.station_value_v_progress = [] - # model time when cost function was evaluated - self.simulation_time = [] - self.model_observation_field = None - self.initialized = False - - def register_observation_data(self, station_names, variable, time, - u, v, x, y, start_times=None, end_times=None): - """ - Add station time series data to the object. - - The `x`, and `y` coordinates must be such that - they allow extraction of model data at the same coordinates. - - :arg list station_names: list of station names - :arg str variable: canonical variable name, e.g. 'elev' - :arg list time: array of time stamps, one for each station - :arg list values: array of observations, one for each station - :arg list x: list of station x coordinates - :arg list y: list of station y coordinates - :kwarg list start_times: optional start times for the observation periods - :kwarg list end_times: optional end times for the observation periods - """ - self.station_names = station_names - self.variable = variable - self.observation_time = time - self.observation_u = u - self.observation_v = v - self.observation_x = x - self.observation_y = y - num_stations = len(station_names) - self._start_times = start_times or -numpy.ones(num_stations)*numpy.inf - self._end_times = end_times or numpy.ones(num_stations)*numpy.inf - - def set_model_field(self, function): - """ - Set the model field that will be evaluated. - """ - self.model_observation_field = function - - def update_stations_in_use(self, t): - """ - Indicate which stations are in use at the current time. - - An entry of unity indicates use, whereas zero indicates disuse. - """ - if not hasattr(self, 'obs_start_times'): - self.construct_evaluator() - in_use = fd.Function(self.fs_points_0d) - in_use.dat.data[:] = numpy.array( - numpy.bitwise_and( - self.obs_start_times <= t, t <= self.obs_end_times - ), dtype=float) - self.indicator_0d.assign(in_use) - - def construct_evaluator(self): - """ - Builds evaluators needed to compute the error functional. - """ - # Create 0D mesh for station evaluation - xy = numpy.array((self.observation_x, self.observation_y)).T - mesh0d = fd.VertexOnlyMesh(self.mesh, xy) - self.fs_points_0d = fd.FunctionSpace(mesh0d, 'DG', 0) - self.obs_values_0d_u = fd.Function(self.fs_points_0d, name='u observations') - self.obs_values_0d_v = fd.Function(self.fs_points_0d, name='v observations') - self.mod_values_0d_u = fd.Function(self.fs_points_0d, name='u model values') - self.mod_values_0d_v = fd.Function(self.fs_points_0d, name='v model values') - self.indicator_0d = fd.Function(self.fs_points_0d, name='station use indicator') - self.indicator_0d.assign(1.0) - self.cost_function_scaling_0d = fd.Constant(0.0, domain=mesh0d) - self.cost_function_scaling_0d.assign(self.cost_function_scaling) - self.station_weight_0d = fd.Function(self.fs_points_0d, name='station-wise weighting') - self.station_weight_0d.assign(1.0) - interp_kw = {} - if numpy.isfinite(self._start_times).any() or numpy.isfinite(self._end_times).any(): - interp_kw.update({'bounds_error': False, 'fill_value': 0.0}) - - # Construct timeseries interpolator - self.station_interpolators_u = [] - self.station_interpolators_v = [] - self.local_station_index = [] - - if len(mesh0d.coordinates.dat.data[:]) > 0: - for i in range(self.fs_points_0d.dof_dset.size): - # loop over local DOFs and match coordinates to observations - # NOTE this must be done manually as VertexOnlyMesh reorders points - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - xy_diff = xy - numpy.array([x_mesh, y_mesh]) - xy_dist = numpy.sqrt(xy_diff[:, 0]**2 + xy_diff[:, 1]**2) - j = numpy.argmin(xy_dist) - self.local_station_index.append(j) - - x, y = xy[j, :] - t = self.observation_time[j] - u = self.observation_u[j] - v = self.observation_v[j] - x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] - - msg = 'bad station location ' \ - f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' - assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg - # create temporal interpolator - ip_u = interp1d(t, u, **interp_kw) - ip_v = interp1d(t, v, **interp_kw) - self.station_interpolators_u.append(ip_u) - self.station_interpolators_v.append(ip_v) - - # Process start and end times for observations - self.obs_start_times = numpy.array([ - self._start_times[i] for i in self.local_station_index - ]) - self.obs_end_times = numpy.array([ - self._end_times[i] for i in self.local_station_index - ]) - - # expressions for cost function - self.misfit_expr_u = self.obs_values_0d_u - self.mod_values_0d_u - self.misfit_expr_v = self.obs_values_0d_v - self.mod_values_0d_v - self.misfit_expr = (self.misfit_expr_u ** 2 + self.misfit_expr_v ** 2) ** (1/2) - self.initialized = True - - def eval_observation_at_time(self, t): - """ - Evaluate observation time series at the given time. - - :arg t: model simulation time - :returns: list of observation time series values at time `t` - """ - self.update_stations_in_use(t) - ip1 = [float(ip(t)) for ip in self.station_interpolators_u] - ip2 = [float(ip(t)) for ip in self.station_interpolators_v] - return ip1, ip2 - - def eval_cost_function(self, t): - """ - Evaluate the cost function. - - Should be called at every export of the forward model. - """ - assert self.initialized, 'Not initialized, call construct_evaluator first.' - assert self.model_observation_field is not None, 'Model field not set.' - self.simulation_time.append(t) - # evaluate observations at simulation time and stash the result - obs_func_u, obs_func_v = fd.Function(self.fs_points_0d), fd.Function(self.fs_points_0d) - obs_func_u.dat.data[:], obs_func_v.dat.data[:] = self.eval_observation_at_time(t) - self.obs_func_u_list.append(obs_func_u) - self.obs_func_v_list.append(obs_func_v) - - # compute square error - self.obs_values_0d_u.assign(obs_func_u) - self.obs_values_0d_v.assign(obs_func_v) - P1_2d = get_functionspace(self.mesh, 'CG', 1) - u_mod = fd.Function(P1_2d, name='u velocity') - v_mod = fd.Function(P1_2d, name='v velocity') - u_mod.project(self.model_observation_field[0]) - v_mod.project(self.model_observation_field[1]) - self.mod_values_0d_u.interpolate(u_mod, ad_block_tag='u observation') - self.mod_values_0d_v.interpolate(v_mod, ad_block_tag='v observation') - s = self.cost_function_scaling_0d * self.indicator_0d * self.station_weight_0d - self.J_misfit = fd.assemble(s * self.misfit_expr ** 2 * fd.dx, ad_block_tag='misfit_eval') - return self.J_misfit - - def dump_time_series(self): - """ - Stores model time series to disk. - - Obtains station time series from the last optimization iteration, - and stores the data to disk. - - The output files are have the format - `{odir}/diagnostic_timeseries_progress_{station_name}_{variable}.hdf5` - - The file contains the simulation time in the `time` array, and the - station name and coordinates as attributes. The time series data is - stored as a 2D (n_iterations, n_time_steps) array. - """ - assert self.station_names is not None - - create_directory(self.output_directory) - tape = get_working_tape() - blocks_u = tape.get_blocks(tag='u observation') - blocks_v = tape.get_blocks(tag='v observation') - ts_data_u = [b.get_outputs()[0].saved_output.dat.data for b in blocks_u] - ts_data_v = [b.get_outputs()[0].saved_output.dat.data for b in blocks_v] - # shape (ntimesteps, nstations) - ts_data_u = numpy.array(ts_data_u) - ts_data_v = numpy.array(ts_data_v) - # append - self.station_value_u_progress.append(ts_data_u) - self.station_value_v_progress.append(ts_data_v) - var = self.variable - for ilocal, iglobal in enumerate(self.local_station_index): - name = self.station_names[iglobal] - # collect time series data, shape (niters, ntimesteps) - ts_u = numpy.array([s[:, ilocal] for s in self.station_value_u_progress]) - ts_v = numpy.array([s[:, ilocal] for s in self.station_value_v_progress]) - fn = f'diagnostic_timeseries_progress_{name}_{var}.hdf5' - fn = os.path.join(self.output_directory, fn) - with h5py.File(fn, 'w') as hdf5file: - hdf5file.create_dataset('u', data=ts_u) - hdf5file.create_dataset('v', data=ts_v) - hdf5file.create_dataset('time', data=numpy.array(self.simulation_time)) - attrs = { - 'x': self.observation_x[iglobal], - 'y': self.observation_y[iglobal], - 'location_name': name, - } - hdf5file.attrs.update(attrs) - - -class RegularizationCalculator(abc.ABC): - """ - Base class for computing regularization terms. - - A derived class should set :attr:`regularization_expr` in - :meth:`__init__`. Whenever the cost function is evaluated, - the ratio of this expression and the total mesh area will - be added. - """ - @abc.abstractmethod - def __init__(self, function, scaling=1.0): - """ - :arg function: Control :class:`Function` - """ - self.scaling = scaling - self.regularization_expr = 0 - self.mesh = function.function_space().mesh() - # calculate mesh area (to scale the cost function) - self.mesh_area = fd.assemble(fd.Constant(1.0, domain=self.mesh) * fd.dx) - self.name = function.name() - - def eval_cost_function(self): - expr = self.scaling * self.regularization_expr / self.mesh_area * fd.dx - return fd.assemble(expr, ad_block_tag="reg_eval") - - -class HessianRegularizationCalculator(RegularizationCalculator): - r""" - Computes the following regularization term for a cost function - involving a control :class:`Function` :math:`f`: - - .. math:: - J = \gamma \| (\Delta x)^2 H(f) \|^2, - - where :math:`H` is the Hessian of field :math:`f`. - """ - def __init__(self, function, gamma, scaling=1.0): - """ - :arg function: Control :class:`Function` - :arg gamma: Hessian penalty coefficient - """ - super().__init__(function, scaling=scaling) - # solvers to evaluate the gradient of the control - P1v_2d = get_functionspace(self.mesh, "CG", 1, vector=True) - P1t_2d = get_functionspace(self.mesh, "CG", 1, tensor=True) - gradient_2d = fd.Function(P1v_2d, name=f"{self.name} gradient") - hessian_2d = fd.Function(P1t_2d, name=f"{self.name} hessian") - self.hessian_calculator = HessianRecoverer2D( - function, hessian_2d, gradient_2d) - - h = fd.CellSize(self.mesh) - # regularization expression |hessian|^2 - # NOTE this is normalized by the mesh element size - # d^2 u/dx^2 * dx^2 ~ du^2 - self.regularization_expr = gamma * fd.inner(hessian_2d, hessian_2d) * h**4 - - def eval_cost_function(self): - self.hessian_calculator.solve() - return super().eval_cost_function() - - -class RSpaceRegularizationCalculator(RegularizationCalculator): - r""" - Computes the following regularization term for a cost function - involving a control :class:`Function` :math:`f` from an R-space: - - .. math:: - J = \gamma (f - f_0)^2, - - where :math:`f_0` is a prior, taken to be the initial value of - :math:`f`. - """ - def __init__(self, function, gamma, eps=1.0e-03, scaling=1.0): - """ - :arg function: Control :class:`Function` - :arg gamma: penalty coefficient - :kwarg eps: tolerance for normalising by near-zero priors - """ - super().__init__(function, scaling=scaling) - R = function.function_space() - if R.ufl_element().family() != "Real": - raise ValueError("function must live in R-space") - prior = fd.Function(R, name=f"{self.name} prior") - prior.assign(function, annotate=False) # Set the prior to the initial value - self.regularization_expr = gamma * (function - prior) ** 2 / ufl.max_value(abs(prior), eps) - # NOTE: If the prior is small then dividing by prior**2 puts too much emphasis - # on the regularization. Therefore, we divide by abs(prior) instead. - - -class ControlRegularizationManager: - """ - Handles regularization of multiple control fields - """ - def __init__(self, function_list, gamma_list, penalty_term_scaling=None, - calculator=HessianRegularizationCalculator): - """ - :arg function_list: list of control functions - :arg gamma_list: list of penalty parameters - :kwarg penalty_term_scaling: Penalty term scaling factor - :kwarg calculator: class used for obtaining regularization - """ - self.reg_calculators = [] - assert len(function_list) == len(gamma_list), \ - 'Number of control functions and parameters must match' - self.reg_calculators = [ - calculator(f, g, scaling=penalty_term_scaling) - for f, g in zip(function_list, gamma_list) - ] - - def eval_cost_function(self): - return sum([r.eval_cost_function() for r in self.reg_calculators]) diff --git a/examples/tohoku_inversion/inverse_problem.py b/examples/tohoku_inversion/inverse_problem.py index 3f53f92e4..f623dada8 100644 --- a/examples/tohoku_inversion/inverse_problem.py +++ b/examples/tohoku_inversion/inverse_problem.py @@ -73,7 +73,7 @@ sta_manager = inversion_tools.StationObservationManager( mesh2d, output_directory=options.output_directory ) -sta_manager.load_observation_data( +sta_manager.load_elev_observation_data( observation_data_dir, station_names, variable, diff --git a/thetis/inversion_tools.py b/thetis/inversion_tools.py index c571d375d..20678bc00 100644 --- a/thetis/inversion_tools.py +++ b/thetis/inversion_tools.py @@ -16,6 +16,24 @@ import os +def group_consecutive_elements(indices): + grouped_list = [] + current_group = [] + + for index in indices: + if not current_group or index == current_group[-1]: + current_group.append(index) + else: + grouped_list.append(current_group) + current_group = [index] + + # Append the last group + if current_group: + grouped_list.append(current_group) + + return grouped_list + + class CostFunctionCallback(DiagnosticCallback): def __init__(self, solver_obj, cost_function, **kwargs): # Disable logging and HDF5 export @@ -72,6 +90,7 @@ def __init__(self, sta_manager, output_dir='outputs', no_exports=False, real=Fal self.output_dir = output_dir self.no_exports = no_exports or real self.real = real + self.maps = [] self.penalty_parameters = penalty_parameters self.cost_function_scaling = cost_function_scaling or fd.Constant(1.0) self.sta_manager.cost_function_scaling = self.cost_function_scaling @@ -79,6 +98,7 @@ def __init__(self, sta_manager, output_dir='outputs', no_exports=False, real=Fal self.test_gradient = test_gradient self.outfiles_m = [] self.outfiles_dJdm = [] + self.outfile_index = [] self.control_exporters = [] self.initialized = False @@ -98,6 +118,7 @@ def __init__(self, sta_manager, output_dir='outputs', no_exports=False, real=Fal self.nb_grad_evals = 0 self.control_coeff_list = [] self.control_list = [] + self.control_family = [] def initialize(self): if not self.no_exports: @@ -105,29 +126,58 @@ def initialize(self): raise ValueError("Exports are not supported in Real mode.") create_directory(self.output_dir) create_directory(self.output_dir + '/hdf5') - for i in range(len(self.control_coeff_list)): + for i in range(max(self.outfile_index)+1): self.outfiles_m.append( fd.VTKFile(f'{self.output_dir}/control_progress_{i:02d}.pvd')) self.outfiles_dJdm.append( fd.VTKFile(f'{self.output_dir}/gradient_progress_{i:02d}.pvd')) self.initialized = True - def add_control(self, f): + def add_control(self, f, mapping=None, new_map=False): """ Add a control field. Can be called multiple times in case of multiparameter optimization. :arg f: Function or Constant to be used as a control variable. + :kwarg mapping: map that dictates how each Control relates to the function being altered + :kwarg new_map: if True, create a new exporter """ + if len(self.control_coeff_list) == 0: + new_map = True + if mapping: + self.maps.append(mapping) + else: + self.maps.append(None) self.control_coeff_list.append(f) self.control_list.append(Control(f)) + function_space = f.function_space() + element = function_space.ufl_element() + family = element.family() + self.control_family.append(family) if isinstance(f, fd.Function) and not self.no_exports: j = len(self.control_coeff_list) - 1 prefix = f'control_{j:02d}' - self.control_exporters.append( - HDF5Exporter(f.function_space(), self.output_dir + '/hdf5', prefix) - ) + if family == 'Real': # i.e. Control is a Constant (assigned to a mesh, so won't register as one) + if new_map: + self.control_exporters.append( + HDF5Exporter(get_functionspace(self.sta_manager.mesh, 'CG', 1), self.output_dir + '/hdf5', + prefix) + ) + if len(self.control_coeff_list) == 1: + self.outfile_index.append(0) + else: + self.outfile_index.append(self.outfile_index[-1] + 1) + else: + self.outfile_index.append(self.outfile_index[-1]) + else: # i.e. Control is a Function + self.control_exporters.append( + HDF5Exporter(function_space, self.output_dir + '/hdf5', prefix) + ) + if len(self.control_coeff_list) == 1: + self.outfile_index.append(0) + else: + self.outfile_index.append(self.outfile_index[-1] + 1) def reset_counters(self): self.nb_grad_evals = 0 @@ -172,7 +222,7 @@ def update_progress(self): toc = self.stop_clock() if self.i == 0: for f in self.control_coeff_list: - print_function_value_range(f, prefix='Initial') + print_function_value_range(f, name=f.name, prefix='Initial') elapsed = '-' if self.tic is None else f'{toc - self.tic:.1f} s' self.tic = toc @@ -206,21 +256,54 @@ def update_progress(self): f'grad_ev={self.nb_grad_evals}, duration {elapsed}') if not self.no_exports: - # control output - for j in range(len(self.control_coeff_list)): - m = self.m_list[j] - # vtk format - o = self.outfiles_m[j] - m.rename(self.control_coeff_list[j].name()) - o.write(m) - # hdf5 format + grouped_indices = group_consecutive_elements(self.outfile_index) + ref_index = 0 + for j in range(len(self.outfiles_m)): + m = self.m_list[ref_index] + o_m = self.outfiles_m[j] + o_jm = self.outfiles_dJdm[j] + self.dJdm_list[ref_index].rename('Gradient') e = self.control_exporters[j] - e.export(m) - # gradient output - for f, o in zip(self.dJdm_list, self.outfiles_dJdm): - # store gradient in vtk format - f.rename('Gradient') - o.write(f) + if len(grouped_indices[j]) > 1: + # can only be Constants + mesh2d = self.sta_manager.mesh + P1_2d = get_functionspace(mesh2d, 'CG', 1) + # vtk format + field = fd.Function(P1_2d, name='control') + self.update_n(field, self.m_list[ref_index:ref_index + len(grouped_indices[j])], + ref_index, ref_index + len(grouped_indices[j])) + o_m.write(field) + # hdf5 format + e.export(field) + # gradient output + gradient = fd.Function(P1_2d, name='Gradient') + self.update_n(gradient, self.dJdm_list[ref_index:ref_index + len(grouped_indices[j])], + ref_index, ref_index + len(grouped_indices[j])) + o_jm.write(gradient) + else: + if self.control_family[ref_index] == 'Real': + mesh2d = self.sta_manager.mesh + P1_2d = get_functionspace(mesh2d, 'CG', 1) + # vtk format + field = fd.Function(P1_2d, name='control') + field.assign(domain_constant(m, mesh2d)) + o_m.write(field) + # hdf5 format + e.export(field) + # gradient output + gradient = fd.Function(P1_2d, name='Gradient') + gradient.assign(domain_constant(self.dJdm_list[ref_index], mesh2d)) + o_jm.write(gradient) + else: + # control output + # vtk format + m.rename(self.control_coeff_list[j].name()) + o_m.write(m) + # hdf5 format + e.export(m) + # gradient output + o_jm.write(self.dJdm_list[ref_index]) + ref_index += len(grouped_indices[j]) self.i += 1 self.reset_counters() @@ -278,8 +361,14 @@ def get_cost_function(self, solver_obj, weight_by_variance=False): # in parallel access to .dat.data should be collective if len(var.dat.data[:]) > 0: for i, j in enumerate(self.sta_manager.local_station_index): - var.dat.data[i] = numpy.var(self.sta_manager.observation_values[j]) - self.sta_manager.station_weight_0d.interpolate(1/var) + if self.sta_manager.obs_param == 'elev': + var.dat.data[i] = numpy.var(self.sta_manager.observation_elev[j]) + else: + obs_speed = numpy.sqrt( + numpy.array(self.sta_manager.observation_u[j]) ** 2 + + numpy.array(self.sta_manager.observation_v[j]) ** 2) + var.dat.data[i] = numpy.var(obs_speed) + self.sta_manager.station_weight_0d.interpolate(1 / var) def cost_fn(): t = solver_obj.simulation_time @@ -325,6 +414,25 @@ def optimization_callback(m): return optimization_callback + def update_n(self, n, m, start_index, end_index): + """ + Update the function `n` by adding weighted masks from `m`. + + This method resets `n` to zero and then adds the weighted masks defined in + `self.maps` to `n`. Each element in `m` is multiplied by the corresponding + mask before being added to `n`. + + :param n: Function or Field to be updated. + :param m: List of Constants/Functions to be combined with the masks. + :param start_index: Starting map. + :param end_index: Final map. + :raises ValueError: If the lengths of `m` and `self.maps` do not match. + """ + n.assign(0) + # Add the weighted masks to n + for m_, mask_ in zip(m, self.maps[start_index:end_index]): + n += m_ * mask_ + def minimize(self, opt_method="BFGS", bounds=None, **opt_options): """ Minimize the reduced functional using a given optimization routine. @@ -383,8 +491,14 @@ class StationObservationManager: This object evaluates the model fields at the station locations, interpolates the observations time series to the model time, computes the error functional, and also stores the model's time series data to disk. + + :param observation_param: The type of observation being compared. It can be either + 'elev' for elevation data or 'uv' for velocity data. + Elevation data refers to the surface elevation time series, + while velocity data ('uv') refers to the horizontal velocity + components (u, v) at each station location. """ - def __init__(self, mesh, output_directory='outputs'): + def __init__(self, mesh, output_directory='outputs', observation_param='elev'): """ :arg mesh: the 2D mesh object. :kwarg output_directory: directory where model time series are stored. @@ -396,35 +510,47 @@ def __init__(self, mesh, output_directory='outputs'): self.cost_function_scaling = fd.Constant(1.0) self.output_directory = output_directory # keep observation time series in memory - self.obs_func_list = [] + self.obs_func_elev_list = [] + self.obs_func_u_list = [] + self.obs_func_v_list = [] # keep model time series in memory during optimization progress - self.station_value_progress = [] + self.station_value_elev_progress = [] + self.station_value_u_progress = [] + self.station_value_v_progress = [] # model time when cost function was evaluated self.simulation_time = [] + self.obs_param = observation_param self.model_observation_field = None self.initialized = False def register_observation_data(self, station_names, variable, time, - values, x, y, start_times=None, end_times=None): + x, y, u=None, v=None, elev=None, start_times=None, end_times=None): """ Add station time series data to the object. The `x`, and `y` coordinates must be such that they allow extraction of model data at the same coordinates. + Observation data is either elevation or velocity u, v components. + :arg list station_names: list of station names :arg str variable: canonical variable name, e.g. 'elev' :arg list time: array of time stamps, one for each station - :arg list values: array of observations, one for each station + :arg list elev: array of elevation observations, one for each station :arg list x: list of station x coordinates :arg list y: list of station y coordinates :kwarg list start_times: optional start times for the observation periods :kwarg list end_times: optional end times for the observation periods """ + if elev is None and (u is None or v is None): + raise ValueError("Either 'elev' must be provided, or both 'u' and 'v' must be provided.") + self.station_names = station_names self.variable = variable self.observation_time = time - self.observation_values = values + self.observation_elev = elev + self.observation_u = u + self.observation_v = v self.observation_x = x self.observation_y = y num_stations = len(station_names) @@ -437,12 +563,12 @@ def set_model_field(self, function): """ self.model_observation_field = function - def load_observation_data(self, observation_data_dir, station_names, variable, - start_times=None, end_times=None): + def load_elev_observation_data(self, observation_data_dir, station_names, variable, + start_times=None, end_times=None): """ - Load observation data from disk. + Load elevation observation data from disk. - Assumes that observation data were stored with + Assumes that elevation observation data were stored with `TimeSeriesCallback2D` during the forward run. For generic case, use `register_observation_data` instead. @@ -474,7 +600,7 @@ def load_observation_data(self, observation_data_dir, station_names, variable, observation_x, observation_y = numpy.array(observation_coords).T self.register_observation_data( station_names, variable, observation_time, - observation_values, observation_x, observation_y, + observation_x, observation_y, elev=observation_values, start_times=start_times, end_times=end_times, ) self.construct_evaluator() @@ -502,11 +628,17 @@ def construct_evaluator(self): xy = numpy.array((self.observation_x, self.observation_y)).T mesh0d = fd.VertexOnlyMesh(self.mesh, xy) self.fs_points_0d = fd.FunctionSpace(mesh0d, 'DG', 0) - self.obs_values_0d = fd.Function(self.fs_points_0d, name='observations') - self.mod_values_0d = fd.Function(self.fs_points_0d, name='model values') + if self.obs_param == 'elev': + self.obs_values_0d_elev = fd.Function(self.fs_points_0d, name='elev observations') + self.mod_values_0d_elev = fd.Function(self.fs_points_0d, name='elev model values') + else: + self.obs_values_0d_u = fd.Function(self.fs_points_0d, name='u observations') + self.obs_values_0d_v = fd.Function(self.fs_points_0d, name='v observations') + self.mod_values_0d_u = fd.Function(self.fs_points_0d, name='u model values') + self.mod_values_0d_v = fd.Function(self.fs_points_0d, name='v model values') self.indicator_0d = fd.Function(self.fs_points_0d, name='station use indicator') self.indicator_0d.assign(1.0) - self.cost_function_scaling_0d = domain_constant(0.0, mesh0d) + self.cost_function_scaling_0d = fd.Constant(0.0, domain=mesh0d) self.cost_function_scaling_0d.assign(self.cost_function_scaling) self.station_weight_0d = fd.Function(self.fs_points_0d, name='station-wise weighting') self.station_weight_0d.assign(1.0) @@ -515,8 +647,11 @@ def construct_evaluator(self): interp_kw.update({'bounds_error': False, 'fill_value': 0.0}) # Construct timeseries interpolator - self.station_interpolators = [] + self.station_interpolators_elev = [] + self.station_interpolators_u = [] + self.station_interpolators_v = [] self.local_station_index = [] + if len(mesh0d.coordinates.dat.data[:]) > 0: for i in range(self.fs_points_0d.dof_dset.size): # loop over local DOFs and match coordinates to observations @@ -529,15 +664,23 @@ def construct_evaluator(self): x, y = xy[j, :] t = self.observation_time[j] - v = self.observation_values[j] x_mesh, y_mesh = mesh0d.coordinates.dat.data[i, :] msg = 'bad station location ' \ - f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x-x_mesh} {y-y_mesh}' + f'{j} {i} {x} {x_mesh} {y} {y_mesh} {x - x_mesh} {y - y_mesh}' assert numpy.allclose([x, y], [x_mesh, y_mesh]), msg - # create temporal interpolator - ip = interp1d(t, v, **interp_kw) - self.station_interpolators.append(ip) + if self.obs_param == 'elev': + elev = self.observation_elev[j] + # create temporal interpolator + ip_elev = interp1d(t, elev, **interp_kw) + self.station_interpolators_elev.append(ip_elev) + else: + u = self.observation_u[j] + v = self.observation_v[j] + ip_u = interp1d(t, u, **interp_kw) + ip_v = interp1d(t, v, **interp_kw) + self.station_interpolators_u.append(ip_u) + self.station_interpolators_v.append(ip_v) # Process start and end times for observations self.obs_start_times = numpy.array([ @@ -548,7 +691,13 @@ def construct_evaluator(self): ]) # expressions for cost function - self.misfit_expr = self.obs_values_0d - self.mod_values_0d + if self.obs_param == 'elev': + self.misfit_expr_elev = self.obs_values_0d_elev - self.mod_values_0d_elev + self.misfit_expr = self.misfit_expr_elev + else: + self.misfit_expr_u = self.obs_values_0d_u - self.mod_values_0d_u + self.misfit_expr_v = self.obs_values_0d_v - self.mod_values_0d_v + self.misfit_expr = (self.misfit_expr_u ** 2 + self.misfit_expr_v ** 2) ** (1/2) self.initialized = True def eval_observation_at_time(self, t): @@ -559,7 +708,13 @@ def eval_observation_at_time(self, t): :returns: list of observation time series values at time `t` """ self.update_stations_in_use(t) - return [float(ip(t)) for ip in self.station_interpolators] + if self.obs_param == 'elev': + ip = [float(ip(t)) for ip in self.station_interpolators_elev] + return ip + else: + ip1 = [float(ip(t)) for ip in self.station_interpolators_u] + ip2 = [float(ip(t)) for ip in self.station_interpolators_v] + return ip1, ip2 def eval_cost_function(self, t): """ @@ -571,16 +726,39 @@ def eval_cost_function(self, t): assert self.model_observation_field is not None, 'Model field not set.' self.simulation_time.append(t) # evaluate observations at simulation time and stash the result - obs_func = fd.Function(self.fs_points_0d) - obs_func.dat.data[:] = self.eval_observation_at_time(t) - self.obs_func_list.append(obs_func) - - # compute square error - self.obs_values_0d.assign(obs_func) - self.mod_values_0d.interpolate(self.model_observation_field, ad_block_tag='observation') - s = self.cost_function_scaling_0d * self.indicator_0d * self.station_weight_0d - self.J_misfit = fd.assemble(s * self.misfit_expr ** 2 * fd.dx, ad_block_tag='misfit_eval') - return self.J_misfit + if self.obs_param == 'elev': + obs_func_elev = fd.Function(self.fs_points_0d) + obs_func_elev.dat.data[:] = self.eval_observation_at_time(t) + self.obs_func_elev_list.append(obs_func_elev) + + # compute square error + self.obs_values_0d_elev.assign(obs_func_elev) + P1_2d = get_functionspace(self.mesh, 'CG', 1) + elev_mod = fd.Function(P1_2d, name='elev') + elev_mod.project(self.model_observation_field) + self.mod_values_0d_elev.interpolate(elev_mod, ad_block_tag='elev observation') + s = self.cost_function_scaling_0d * self.indicator_0d * self.station_weight_0d + self.J_misfit = fd.assemble(s * self.misfit_expr ** 2 * fd.dx, ad_block_tag='misfit_eval') + return self.J_misfit + else: + obs_func_u, obs_func_v = fd.Function(self.fs_points_0d), fd.Function(self.fs_points_0d) + obs_func_u.dat.data[:], obs_func_v.dat.data[:] = self.eval_observation_at_time(t) + self.obs_func_u_list.append(obs_func_u) + self.obs_func_v_list.append(obs_func_v) + + # compute square error + self.obs_values_0d_u.assign(obs_func_u) + self.obs_values_0d_v.assign(obs_func_v) + P1_2d = get_functionspace(self.mesh, 'CG', 1) + u_mod = fd.Function(P1_2d, name='u velocity') + v_mod = fd.Function(P1_2d, name='v velocity') + u_mod.project(self.model_observation_field[0]) + v_mod.project(self.model_observation_field[1]) + self.mod_values_0d_u.interpolate(u_mod, ad_block_tag='u observation') + self.mod_values_0d_v.interpolate(v_mod, ad_block_tag='v observation') + s = self.cost_function_scaling_0d * self.indicator_0d * self.station_weight_0d + self.J_misfit = fd.assemble(s * self.misfit_expr ** 2 * fd.dx, ad_block_tag='misfit_eval') + return self.J_misfit def dump_time_series(self): """ @@ -600,28 +778,58 @@ def dump_time_series(self): create_directory(self.output_directory) tape = get_working_tape() - blocks = tape.get_blocks(tag='observation') - ts_data = [b.get_outputs()[0].saved_output.dat.data for b in blocks] - # shape (ntimesteps, nstations) - ts_data = numpy.array(ts_data) - # append - self.station_value_progress.append(ts_data) - var = self.variable - for ilocal, iglobal in enumerate(self.local_station_index): - name = self.station_names[iglobal] - # collect time series data, shape (niters, ntimesteps) - ts = numpy.array([s[:, ilocal] for s in self.station_value_progress]) - fn = f'diagnostic_timeseries_progress_{name}_{var}.hdf5' - fn = os.path.join(self.output_directory, fn) - with h5py.File(fn, 'w') as hdf5file: - hdf5file.create_dataset(var, data=ts) - hdf5file.create_dataset('time', data=numpy.array(self.simulation_time)) - attrs = { - 'x': self.observation_x[iglobal], - 'y': self.observation_y[iglobal], - 'location_name': name, - } - hdf5file.attrs.update(attrs) + if self.obs_param == 'elev': + blocks_elev = tape.get_blocks(tag='elev observation') + ts_data_elev = [b.get_outputs()[0].saved_output.dat.data for b in blocks_elev] + # shape (ntimesteps, nstations) + ts_data_elev = numpy.array(ts_data_elev) + # append + self.station_value_elev_progress.append(ts_data_elev) + var = self.variable + for ilocal, iglobal in enumerate(self.local_station_index): + name = self.station_names[iglobal] + # collect time series data, shape (niters, ntimesteps) + ts_elev = numpy.array([s[:, ilocal] for s in self.station_value_elev_progress]) + fn = f'diagnostic_timeseries_progress_{name}_{var}.hdf5' + fn = os.path.join(self.output_directory, fn) + with h5py.File(fn, 'w') as hdf5file: + hdf5file.create_dataset('elev', data=ts_elev) + hdf5file.create_dataset('time', data=numpy.array(self.simulation_time)) + attrs = { + 'x': self.observation_x[iglobal], + 'y': self.observation_y[iglobal], + 'location_name': name, + } + hdf5file.attrs.update(attrs) + else: + blocks_u = tape.get_blocks(tag='u observation') + blocks_v = tape.get_blocks(tag='v observation') + ts_data_u = [b.get_outputs()[0].saved_output.dat.data for b in blocks_u] + ts_data_v = [b.get_outputs()[0].saved_output.dat.data for b in blocks_v] + # shape (ntimesteps, nstations) + ts_data_u = numpy.array(ts_data_u) + ts_data_v = numpy.array(ts_data_v) + # append + self.station_value_u_progress.append(ts_data_u) + self.station_value_v_progress.append(ts_data_v) + var = self.variable + for ilocal, iglobal in enumerate(self.local_station_index): + name = self.station_names[iglobal] + # collect time series data, shape (niters, ntimesteps) + ts_u = numpy.array([s[:, ilocal] for s in self.station_value_u_progress]) + ts_v = numpy.array([s[:, ilocal] for s in self.station_value_v_progress]) + fn = f'diagnostic_timeseries_progress_{name}_{var}.hdf5' + fn = os.path.join(self.output_directory, fn) + with h5py.File(fn, 'w') as hdf5file: + hdf5file.create_dataset('u', data=ts_u) + hdf5file.create_dataset('v', data=ts_v) + hdf5file.create_dataset('time', data=numpy.array(self.simulation_time)) + attrs = { + 'x': self.observation_x[iglobal], + 'y': self.observation_y[iglobal], + 'location_name': name, + } + hdf5file.attrs.update(attrs) class RegularizationCalculator(abc.ABC): @@ -642,7 +850,7 @@ def __init__(self, function, scaling=1.0): self.regularization_expr = 0 self.mesh = function.function_space().mesh() # calculate mesh area (to scale the cost function) - self.mesh_area = fd.assemble(fd.Constant(1.0) * fd.dx(domain=self.mesh)) + self.mesh_area = fd.assemble(fd.Constant(1.0, domain=self.mesh) * fd.dx) self.name = function.name() def eval_cost_function(self): From a63c4d084e0f47fd877e7e2446456448cdbe17e5 Mon Sep 17 00:00:00 2001 From: Connor Jordan Date: Mon, 16 Sep 2024 20:33:37 +0100 Subject: [PATCH 15/15] Add initial time as hdf5 attribute in callbacks - This is easier to directly load than the units 'time since initial time' - still keep the units --- thetis/callback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/thetis/callback.py b/thetis/callback.py index 52195e926..1efa51e76 100644 --- a/thetis/callback.py +++ b/thetis/callback.py @@ -209,6 +209,7 @@ def __init__(self, solver_obj, array_dim=1, attrs=None, if init_date is not None and include_time: time_units = 'seconds since ' + init_date.isoformat() self.var_attrs['time'] = {'units': time_units} + self.attrs['simulation_initial_date'] = init_date.isoformat() def set_write_mode(self, mode): """