from time import sleep
from dataclasses import dataclass
from timeit import default_timer as timer
import shutil
from glob import glob
import h5py
import numpy as np
import turbigen.grid
from turbigen.exceptions import ConvergenceError
import subprocess
import os
from pathlib import Path
import signal
import sys
import re
import grp
import getpass
from copy import copy
from turbigen.solvers.base import BaseSolver, ConvergenceHistory
import turbigen.util
logger = turbigen.util.make_logger()
[docs]
@dataclass
class Config(BaseSolver):
# Override base attributes
_name = "ts3"
workdir: Path = None
"""Working directory to run the simulation in."""
environment_script: Path = Path(
"/usr/local/software/turbostream/ts3610_a100/bashrc_module_ts3610_a100"
)
"""Setup environment shell script to be sourced before running."""
atol_eta: float = 0.005
"""Absolute tolerance on drift in isentropic efficiency."""
cfl: float = 0.4
"""Courant--Friedrichs--Lewy number, reduce for more stability."""
dampin: float = 25.0
"""Negative feedback factor, reduce for more stability."""
facsecin: float = 0.005
"""Fourth-order smoothing feedback factor, increase for more stability."""
fmgrid: float = 0.2
"""Multigrid factor, reduce for more stability."""
ilos: int = 2
"""Viscous model, 0 for inviscid, 1 for mixing-length, 2 for Spalart-Allmaras."""
Lref_xllim: str = "pitch"
"""Mixing length characteristic dimension, "pitch" or "span"."""
nchange: int = 2000
"""At start of simulation, ramp smoothing and damping over this many time steps."""
nstep: int = 10000
"""Number of time steps."""
nstep_avg: int = 5000
"""Average over the last `nstep_avg` steps of the calculation."""
rfmix: float = 0.0
"""Mixing plane relaxation factor."""
rtol_mdot: float = 0.01
"""Relative tolerance on mass flow conservation error and drift."""
sfin: float = 0.5
"""Proportion of second-order smoothing, increase for more stability."""
tvr: float = 10.0
"""Initial guess of turbulent viscosity ratio."""
xllim: float = 0.03
"""Mixing length limit as a fraction of characteristic dimension."""
rfin: float = 0.5
"""Inlet relaxation factor, reduce for low-Mach flows."""
nstep_soft: int = 0
"""Number of steps for soft start precursor simulation."""
sa_helicity_option: int = 0
"""Spalart--Allmaras turbulence model helicity correction."""
smooth_scale_dts_option: int = 0
show_yplus: bool = False
laminar: bool = False
"""Enable laminar boundary layers on all walls."""
fac_st0: float = 1.0
ipout: int = 3
convert_sliding: bool = False
precon: int = 0
dts: int = 0
nstep_cycle: int = 72
nstep_inner: int = 200
ncycle: int = 0
frequency: float = 0.0
nstep_save_probe: int = 0
nstep_save_start_probe: int = 0
xllim_free: float = 0.1
free_turb: float = 0.05
turbvis_lim: float = 3000.0
sa_ch1: float = 0.71
sa_ch2: float = 0.6
def application_variables(self, ga, cp, mu):
# """Make a complete set of applications variables, with defaults overriden
av = DEFAULT_AV.copy()
for k in av:
if hasattr(self, k):
av[k] = getattr(self, k)
av["ga"] = ga
av["cp"] = cp
av["viscosity"] = mu
if av["dts"]:
av["nstep_save_start"] = av["ncycle"] * av["nstep_cycle"] - self.nstep_avg
else:
if av["nstep"] < 0:
av["nstep"] = int(2 * self.nstep_avg)
av["nstep_save_start"] = av["nstep"] - self.nstep_avg
# Raise error if averaging not OK
nstep = av["nstep"]
nstep_save_start = av["nstep_save_start"]
if nstep_save_start >= nstep and nstep > 0 and not av["dts"]:
raise Exception(f"nstep_save_start={nstep_save_start} is > nstep={nstep}")
return av
def block_variables(self, block, rref, laminar):
# """Make a dictionary of block variables, with defaults overriden as needed."""
bv = DEFAULT_BV.copy()
for k in bv:
if hasattr(self, k):
bv[k] = getattr(self, k)
# Set some block variables using the block data
bv["nblade"] = block.Nb
bv["fblade"] = float(block.Nb)
assert np.ptp(block.rpm) == 0.0
rpm = block.rpm.mean()
if self.convert_sliding and rpm == 0.0:
for patch in block.patches:
if isinstance(patch, turbigen.grid.MixingPatch):
assert np.ptp(patch.match.block.rpm) == 0.0
rpm = patch.match.block.rpm.mean()
break
bv["rpm"] = rpm
bv["fracann"] = 1.0 / float(block.Nb)
if self.Lref_xllim == "pitch":
bv["xllim"] = 2.0 * np.pi * rref / float(block.Nb) * self.xllim
elif self.Lref_xllim == "span":
span = np.ptp(block.r)
bv["xllim"] = span * self.xllim
elif self.Lref_xllim == "fix":
bv["xllim"] = self.xllim
else:
raise Exception(f"Unrecognised Lref_xllim={self.Lref_xllim}")
bv.update(_get_wall_rpms(block))
if laminar:
# Set laminar from i=0 to i=ni on every block
ni1 = block.ni - 1
bv["itrans"] = -1
bv["itrans_j1_st"] = 0
bv["itrans_j1_en"] = ni1
bv["itrans_j2_st"] = 0
bv["itrans_j2_en"] = ni1
bv["itrans_k1_st"] = 0
bv["itrans_k1_en"] = ni1
bv["itrans_k2_st"] = 0
bv["itrans_k2_en"] = ni1
bv["itrans_j1_frac"] = 0.1
bv["itrans_j2_frac"] = 0.1
bv["itrans_k1_frac"] = 0.1
bv["itrans_k2_frac"] = 0.1
return bv
def _robust(self):
"""Increase damping and smoothing, lower CFL, and use mixing-length model."""
c = copy(self)
c.ilos = 1
c.dampin = 3.0
c.facsecin = 0.02
c.sfin = 2.0
c.cfl = 0.3
c.fmgrid = 0.0
c.soft_start = False
c.precon = 0
c.dts = 0
if c.nstep_soft:
c.nstep = c.nstep_soft
return c
# Block attributes that must be present
# Where we should not set a default, use None
DEFAULT_BA = {
"bid": None,
"nc": 0,
"np": None,
"ncl": 0,
"ni": None,
"nj": None,
"nk": None,
"procid": 0,
"threadid": 0,
}
# Patch attributes that must be present
# Where we should not set a default, use None
DEFAULT_PA = {
"pid": None,
"bid": None,
"ist": None,
"jst": None,
"kst": None,
"ien": None,
"jen": None,
"ken": None,
"idir": None,
"jdir": None,
"kdir": None,
"kind": None,
"nface": 0,
"nt": 1,
"nxbid": None,
"nxpid": None,
}
# Application variables that must be present
# Where we should not set a default, use None
DEFAULT_AV = {
"adaptive_smoothing": 1,
"cfl": 0.4,
"cfl_en_ko": 0.4,
"cfl_ko": 0.4,
"cfl_st_ko": 0.01,
"cp": None,
"cp0_0": 1005.0,
"cp0_1": 1005.0,
"cp1_0": 0.0,
"cp1_1": 0.0,
"cp2_0": 0.0,
"cp2_1": 0.0,
"cp3_0": 0.0,
"cp3_1": 0.0,
"cp4_0": 0.0,
"cp4_1": 0.0,
"cp5_0": 0.0,
"cp5_1": 0.0,
"dampin": 25.0,
"dts": 0,
"dts_conv": 0.0,
"fac_sa_smth": 4.0,
"fac_sa_step": 1.0,
"fac_st0": 1.0,
"fac_st0_option": 0,
"fac_st1": 1.0,
"fac_st2": 1.0,
"fac_st3": 1.0,
"fac_stmix": 0.0,
"fac_wall": 1.0,
"facsafe": 0.2,
"facsecin": 0.005,
"frequency": 0.0,
"ga": None,
"if_ale": 0,
"if_no_mg": 0,
"ifgas": 0,
"ifsuperfac": 0,
"ilos": 2,
"ko_dist": 1e-4,
"ko_restart": 0,
"nchange": 1000,
"ncycle": 0,
"nlos": 5,
"nomatch_int": 1,
"nspecies": 1,
"nstep": 10000,
"nstep_cycle": 0,
"nstep_inner": 0,
"nstep_save": 0,
"nstep_save_probe": 0,
"nstep_save_start": 0,
"nstep_save_start_probe": 0,
"poisson_cfl": 0.7,
"poisson_limit": 0,
"poisson_nsmooth": 10,
"poisson_nstep": 0,
"poisson_restart": 2,
"poisson_sfin": 0.02,
"prandtl": 1.0,
"precon": 0,
"pref": 1e5,
"restart": 1,
"rfmix": 0.0,
"rfvis": 0.2,
"rg_cp0": 1005.0,
"rg_cp1": 0.0,
"rg_cp2": 0.0,
"rg_cp3": 0.0,
"rg_cp4": 0.0,
"rg_cp5": 0.0,
"rg_rgas": 287.15,
"rgas_0": 287.15,
"rgas_1": 287.15,
"sa_ch1": 1.0,
"sa_ch2": 1.0,
"sa_helicity_option": 0,
"schmidt_0": 1.0,
"schmidt_1": 1.0,
"sf_scalar": 0.05,
"sfin": 0.5,
"sfin_ko": 0.05,
"sfin_sa": 0.05,
"smooth_scale_directional_option": 0,
"smooth_scale_dts_option": 0,
"smooth_scale_precon_option": 0,
"tref": 300.0,
"turb_vis_damp": 1.0,
"turbvis_lim": 3000.0,
"use_temperature_sensor": 0,
"viscosity": None,
"viscosity_a1": 0.0,
"viscosity_a2": 0.0,
"viscosity_a3": 0.0,
"viscosity_a4": 0.0,
"viscosity_a5": 0.0,
"viscosity_law": 0,
"wall_law": 0,
"write_egen": 0,
"write_force": 0,
"write_tdamp": 0,
"write_yplus": 1,
}
# Block variables that must be present
# Where we should not set a default, use None
DEFAULT_BV = {
"dampin_mul": 1.0,
"fac_st0": 1.0,
"facsecin_mul": 1.0,
"fblade": None,
"fl_ibpa": 0.0,
"fmgrid": 0.2,
"fracann": None,
"free_turb": 0.05,
"fsturb": 1.0,
"ftype": 0,
"itrans": 0,
"itrans_j1_en": 0,
"itrans_j1_frac": 0.0,
"itrans_j1_st": 0,
"itrans_j2_en": 0,
"itrans_j2_frac": 0.0,
"itrans_j2_st": 0,
"itrans_k1_en": 0,
"itrans_k1_frac": 0.0,
"itrans_k1_st": 0,
"itrans_k2_en": 0,
"itrans_k2_frac": 0.0,
"itrans_k2_st": 0,
"jtrans": 0,
"jtrans_i1_en": 0,
"jtrans_i1_frac": 0.0,
"jtrans_i1_st": 0,
"jtrans_i2_en": 0,
"jtrans_i2_frac": 0.0,
"jtrans_i2_st": 0,
"jtrans_k1_en": 0,
"jtrans_k1_frac": 0.0,
"jtrans_k1_st": 0,
"jtrans_k2_en": 0,
"jtrans_k2_frac": 0.0,
"jtrans_k2_st": 0,
"ktrans": 0,
"ktrans_i1_en": 0,
"ktrans_i1_frac": 0.0,
"ktrans_i1_st": 0,
"ktrans_i2_en": 0,
"ktrans_i2_frac": 0.0,
"ktrans_i2_st": 0,
"ktrans_j1_en": 0,
"ktrans_j1_frac": 0.0,
"ktrans_j1_st": 0,
"ktrans_j2_en": 0,
"ktrans_j2_frac": 0.0,
"ktrans_j2_st": 0,
"nblade": None,
"ndup_phaselag": 1,
"nimixl": 0,
"poisson_fmgrid": 0.0,
"pstatin": 800000.0,
"pstatout": 800000.0,
"rpm": None,
"rpmi1": None,
"rpmi2": None,
"rpmj1": None,
"rpmj2": None,
"rpmk1": None,
"rpmk2": None,
"sfin_mul": 1.0,
"srough_i0": 0.0,
"srough_i1": 0.0,
"srough_j0": 0.0,
"srough_j1": 0.0,
"srough_k0": 0.0,
"srough_k1": 0.0,
"superfac": 0.0,
"tstagin": 1200.0,
"tstagout": 1200.0,
"turb_intensity": 5.0,
"vgridin": 50.0,
"vgridout": 50.0,
"xllim": None,
"xllim_free": 0.1,
}
# Patch variables that must be present on inlet patches
# Where we should not set a default, use None
DEFAULT_INLET_PV = {
"rfin": 0.5,
"sfinlet": 0.0,
}
# Patch variables that must be present on outlet patches
# Where we should not set a default, use None
DEFAULT_OUTLET_PV = {
"ipout": 3,
"throttle_type": 0,
"throttle_target": 0.0,
"throttle_k0": 0.0,
"throttle_k1": 0.0,
"throttle_k2": 0.0,
"fthrottle": 0.0,
"pout": None,
"pout_st": 0.0,
"pout_en": 0.0,
"pout_nchange": 0,
}
# Patch variables that must be present on outlet patches
# Where we should not set a default, use None
DEFAULT_POROUS_PV = {"porous_fac_loss": 1.0, "porous_rf": 0.9}
def _get_patch_kind(patch):
"""Choose TS3 patch kind integer based on patch class."""
if isinstance(patch, turbigen.grid.InletPatch):
return 0
elif isinstance(patch, turbigen.grid.OutletPatch):
if patch.force:
return 19 # for 'outlet2d'
else:
return 1
elif isinstance(patch, turbigen.grid.MixingPatch):
return 2
elif isinstance(patch, turbigen.grid.PorousPatch):
return 17
elif isinstance(patch, turbigen.grid.PeriodicPatch):
if patch.cartesian:
return 16
else:
return 5
elif isinstance(patch, turbigen.grid.InviscidPatch):
return 7
elif isinstance(patch, turbigen.grid.ProbePatch):
return 8
elif isinstance(patch, turbigen.grid.NonMatchPatch):
return 15
elif isinstance(patch, turbigen.grid.CoolingPatch):
return 6
else:
raise Exception(f"No TS3 patch kind defined for {patch}")
def _get_patch_sten(patch):
"""Patch attributes describing patch start and end indices."""
# Convert negative indices to zero indexed
ijk_lim = patch.ijk_limits.copy()
nijk = np.reshape(patch.block.shape, (3, 1))
ijk_lim[ijk_lim < 0] = (nijk + ijk_lim)[ijk_lim < 0]
# Add one to end indices to make them exclusive
ijk_lim[:, 1] += 1
# Return as a dictionary of patch attributes
keys = ["ist", "ien", "jst", "jen", "kst", "ken"]
return dict(zip(keys, ijk_lim.flat))
def _get_patch_connectivity(patch, rtol=1e-4):
"""Patch attributes describing periodic or mixing connectivity."""
# Deal with mixing patches
if isinstance(patch, turbigen.grid.MixingPatch):
pm = patch.match
# Find ids for the connected block and patch
nxbid = pm.block.grid.index(pm.block)
not_rot = [
p
for p in pm.block.patches
if not isinstance(p, turbigen.grid.RotatingPatch)
]
nxpid = not_rot.index(pm)
if not patch.slide:
return {"idir": 0, "jdir": 0, "kdir": 0, "nxbid": nxbid, "nxpid": nxpid}
else:
return {
"kind": 3,
"idir": 0,
"jdir": 0,
"kdir": 0,
"nface": 1,
"nxbid": nxbid,
"nxpid": nxpid,
"slide_nxbid": nxbid,
"slide_nxpid": nxpid,
}
is_periodic = isinstance(patch, turbigen.grid.PeriodicPatch)
is_nonmatch = isinstance(patch, turbigen.grid.NonMatchPatch)
is_porous = isinstance(patch, turbigen.grid.PorousPatch)
if is_periodic or is_nonmatch or is_porous:
pm = patch.match
# Find ids for the connected block and patch
nxbid = pm.block.grid.index(pm.block)
not_rot = [
p
for p in pm.block.patches
if not isinstance(p, turbigen.grid.RotatingPatch)
]
nxpid = not_rot.index(pm)
return {
"idir": patch.idir,
"jdir": patch.jdir,
"kdir": patch.kdir,
"nxbid": nxbid,
"nxpid": nxpid,
}
else:
# If the patch is not mixing and not periodic, set dummy values
return {"idir": 0, "jdir": 0, "kdir": 0, "nxbid": 0, "nxpid": 0}
def _block_attributes(block, bid):
"""Make a dictionary of block attributes."""
ba = DEFAULT_BA.copy()
ba["bid"] = bid
ba["ni"], ba["nj"], ba["nk"] = block.shape
ba["np"] = len(
[p for p in block.patches if not isinstance(p, turbigen.grid.RotatingPatch)]
)
return ba
def _patch_attributes(patch, bid, pid):
"""Make a dictionary of patch attributes."""
pa = DEFAULT_PA.copy()
pa["pid"] = pid
pa["bid"] = bid
pa["kind"] = _get_patch_kind(patch)
pa.update(_get_patch_sten(patch))
pa.update(_get_patch_connectivity(patch))
return pa
def _patch_variables(patch, ts3_config):
"""Make a dictionary of patch attributes."""
pv = {}
if isinstance(patch, turbigen.grid.InletPatch):
pv.update(DEFAULT_INLET_PV)
pv["rfin"] = float(patch.rfin)
elif isinstance(patch, turbigen.grid.PorousPatch):
pv.update(DEFAULT_POROUS_PV)
pv["porous_fac_loss"] = float(patch.porous_fac_loss)
elif isinstance(patch, turbigen.grid.ProbePatch):
pv["probe_append"] = 1
elif isinstance(patch, turbigen.grid.OutletPatch):
pv.update(DEFAULT_OUTLET_PV)
pv["pout"] = float(patch.Pout)
pv["ipout"] = ts3_config.ipout
if patch.mdot_target:
pv["throttle_type"] = 1
pv["throttle_target"] = float(patch.mdot_target)
# Turbostream uses 'PDI' control, not 'PID'
pv["throttle_k0"], pv["throttle_k2"], pv["throttle_k1"] = patch.Kpid
if patch.force:
pv.pop("pout")
elif isinstance(patch, turbigen.grid.CoolingPatch):
patch.check()
pv.update(
{
"cool_mass": patch.cool_mass,
"cool_pstag": patch.cool_pstag,
"cool_tstag": patch.cool_tstag,
"cool_type": patch.cool_type,
"cool_angle_def": patch.cool_angle_def,
"cool_sangle": patch.cool_sangle,
"cool_xangle": patch.cool_xangle,
"cool_frac_area": 1.0,
"cool_mach": patch.cool_mach,
}
)
return pv
def _patch_properties(patch):
"""Make a dictionary of patch properties."""
pp = {}
if isinstance(patch, turbigen.grid.InletPatch):
if patch.state.shape == ():
x = np.ones_like(patch.get_cut().x)
pp["pstag"] = patch.state.P * x
# So that the TS3->TS4 converter works for real gases
pp["tstag"] = patch.state.h / patch.state.cp * x
pp["pitch"] = patch.Beta * x
pp["yaw"] = patch.Alpha * x
pp["fsturb_mul"] = x
else:
raise NotImplementedError("Inlet shape not supported")
elif isinstance(patch, turbigen.grid.OutletPatch):
if patch.force:
pp["pout"] = patch.Pout * np.ones_like(patch.get_cut().x)
return pp
def _write_variable(group, name, suffix, val):
"""Save a scalar to an hdf5 file."""
key = name + suffix
if isinstance(val, int):
dtype = np.dtype("i4")
else:
dtype = np.dtype("f4")
if val is None:
raise Exception(f"Unspecified value for variable {name}")
try:
group.create_dataset(key, data=np.reshape(val, (1,)), dtype=dtype)
except Exception:
raise Exception(f"Could not write key={key}, val={val}")
def _write_property(group, name, suffix, val, flat=False):
"""Save an array to an hdf5 file."""
key = name + suffix
dtype = np.dtype("f4")
if val is None:
raise Exception(f"Unspecified value for variable {name}")
if np.isnan(val).any():
raise Exception(f"NaN in variable {name}")
val_out = np.ones(val.shape, dtype=np.float32)
val_out.flat = val.transpose().flat
if flat:
val_out = val_out.flatten()
group.create_dataset(key, data=val_out, dtype=dtype)
def _get_wall_rpms(block):
"""Dictionary of block variables describing wall rotations."""
keys = [
"rpmi1",
"rpmj1",
"rpmk1",
"rpmi2",
"rpmj2",
"rpmk2",
]
vals = np.zeros((6,))
for patch in block.rotating_patches:
st_set = np.logical_and(
patch.ijk_limits[:, 0] == 0, patch.ijk_limits[:, 1] == 0
)
en_set = np.logical_and(
patch.ijk_limits[:, 0] == -1, patch.ijk_limits[:, 1] == -1
)
rpm_patch = patch.Omega * 30.0 / np.pi
vals[:3][st_set] = rpm_patch
vals[3:][en_set] = rpm_patch
# assert block.rpm.ptp() == 0.0
# vals *= block.rpm.mean()
return dict(zip(keys, vals))
def _write_hdf5(grid, ts3_config, fname="input.hdf5"):
"""Using a given configuration, write grid object to an hdf5."""
# Store old internal energy datum
# Then set to zero as assumed by TS
Tu0_old = grid[0].Tu0
for b in grid:
b.set_Tu0(0.0)
for p in grid.inlet_patches:
p.state.set_Tu0(0.0)
# Determine reference radii for mixing length limit
rref = np.empty((grid.nrow,))
for irow, row_block in enumerate(grid.row_blocks):
rref[irow] = np.mean([0.5 * (b.r.max() + b.r.min()) for b in row_block])
input_file_path = os.path.join(ts3_config.workdir, fname)
if not os.path.exists(ts3_config.workdir):
raise Exception(f"Working directory {ts3_config.workdir} does not exist.")
f = h5py.File(input_file_path, "w")
# Get gas properties from the inlet
So1 = grid.inlet_patches[0].state
cp = float(So1.cp)
ga = float(So1.gamma)
mu = float(So1.mu)
grid.inlet_patches[0].rfin = ts3_config.rfin
# Grid attributes
nb = len(grid)
f.attrs["nb"] = nb
f.attrs["ntb"] = 0
# Application variables
for name, val in ts3_config.application_variables(ga, cp, mu).items():
_write_variable(f, name, "_av", val)
procids = grid.partition(ts3_config.ntask)
# Loop over blocks
for ib in range(nb):
key = f"block{ib}"
block = grid[ib]
# Make group to hold block data
block_group = f.create_group(key)
# Block attributes
block_group.attrs.update(_block_attributes(block, ib))
block_group.attrs["procid"] = procids[ib]
# Block variables
if ts3_config.Lref_xllim == "pitch":
rref_block = rref[grid.row_index(block)]
else:
rref_block = np.nan
for name, val in ts3_config.block_variables(
block, rref_block, ts3_config.laminar
).items():
_write_variable(block_group, name, "_bv", val)
# Block properties
_write_property(block_group, "x", "_bp", block.x)
_write_property(block_group, "r", "_bp", block.r)
_write_property(block_group, "rt", "_bp", block.rt)
_write_property(block_group, "ro", "_bp", block.rho)
_write_property(block_group, "rovx", "_bp", block.rhoVx)
_write_property(block_group, "rovr", "_bp", block.rhoVr)
_write_property(block_group, "rorvt", "_bp", block.rhorVt)
_write_property(block_group, "roe", "_bp", block.rhoe)
_write_property(block_group, "phi", "_bp", block.w)
if np.isnan(block.mu_turb).all():
turb_visc = ts3_config.tvr * np.ones_like(block.x) * mu
_write_property(block_group, "trans_dyn_vis", "_bp", turb_visc)
else:
_write_property(block_group, "trans_dyn_vis", "_bp", block.mu_turb)
# Loop over patches
ip = 0
for patch in block.patches:
# Skip rotating patches - set wall rpms using block variables
if isinstance(patch, turbigen.grid.RotatingPatch):
logger.debug(f"Skipping patch {patch}, ip={ip}")
continue
else:
logger.debug(f"Writing patch {patch}, ip={ip}")
# Make group to hold patch data
patch_key = f"patch{ip}"
patch_group = block_group.create_group(patch_key)
# Patch attributes
pa = _patch_attributes(patch, ib, ip)
if "slide_nxbid" in pa:
pv_slide = {k: pa.pop(k) for k in ("slide_nxbid", "slide_nxpid")}
if ts3_config.convert_sliding:
for name, val in pv_slide.items():
_write_variable(patch_group, name, "", val)
else:
pa["kind"] = 2
# Patch variables
for name, val in _patch_variables(patch, ts3_config).items():
_write_variable(patch_group, name, "_pv", val)
# Patch properties
for name, val in _patch_properties(patch).items():
# Make boundary conditions unsteady if needed
if isinstance(patch, turbigen.grid.InletPatch):
if force := patch.force:
t = _get_time_vector(ts3_config)
nt = len(t)
F = 1.0 + patch.amplitude * np.sin(
2.0 * np.pi * ts3_config.frequency * t + patch.phase
).reshape(1, 1, 1, nt)
ga = patch.state.gamma
if force == "isentropic":
Po_Poav = F
To_Toav = Po_Poav ** ((ga - 1.0) / ga)
elif force == "entropic":
Po_Poav = np.ones_like(F)
To_Toav = F
else:
raise Exception(f"Unknown inlet forcing type {force}")
val = np.expand_dims(val, 3)
if name == "pstag":
val = val * Po_Poav
elif name == "tstag":
val = val * To_Toav
else:
val = np.tile(val, (1, 1, 1, nt))
pa["nt"] = nt
if isinstance(patch, turbigen.grid.OutletPatch):
if patch.force:
t = _get_time_vector(ts3_config)
F = 1.0 + patch.amplitude * np.sin(
2.0 * np.pi * ts3_config.frequency * t + patch.phase
).reshape(1, 1, 1, -1)
val = np.expand_dims(val, 3) * F
pa["nt"] = len(t)
_write_property(patch_group, name, "_pp", val, flat=True)
patch_group.attrs.update(pa)
ip += 1
assert ip == block_group.attrs["np"]
# Now check that patch and block ids are consistent
logger.debug("Checking np")
for ib in range(nb):
blk = f[f"block{ib}"]
nptch = blk.attrs["np"]
logger.debug(f"bid={ib}, np={nptch}, len(patches)={len(grid[ib].patches)}")
for ip in range(nptch):
pch = blk[f"patch{ip}"]
bid = pch.attrs["bid"]
pid = pch.attrs["pid"]
nxbid = pch.attrs["nxbid"]
nxblk = f[f"block{nxbid}/"]
nxpid = pch.attrs["nxpid"]
logger.debug(
f'ip={ip}, kind={pch.attrs["kind"]}, '
f"bid={bid}, pid={pid}, "
f'nxbid={nxbid}, nxpid={nxpid}, nxnp={nxblk.attrs["np"]}'
)
ni = blk.attrs["ni"]
nj = blk.attrs["nj"]
nk = blk.attrs["nk"]
ist = pch.attrs["ist"]
jst = pch.attrs["jst"]
kst = pch.attrs["kst"]
ien = pch.attrs["ien"]
jen = pch.attrs["jen"]
ken = pch.attrs["ken"]
assert ist < ni
assert ien < (ni + 1)
assert jst < nj
assert jen < (nj + 1)
assert kst < nk
assert ken < (nk + 1)
assert nxbid < nb
assert nxpid < nxblk.attrs["np"]
f.close()
# Reset the internal energy datum
for b in grid:
b.set_Tu0(Tu0_old)
for p in grid.inlet_patches:
p.state.set_Tu0(Tu0_old)
def _execute(ts3_config):
"""Using a given configuration, execute TS3."""
# Store old working directory and change to this config's
old_workdir = os.getcwd()
os.chdir(ts3_config.workdir)
if not os.path.exists(ts3_config.environment_script):
raise Exception(
f"""Could not locate TS3 env script {ts3_config.environment_script}
Are you on a HPC compute node gpu-q-* (not a login node)?
If you have recently been added to the turbostream user group, log out
and then back in to refresh your access permissions.
"""
)
# Open a subshell, source the environment and run the solver
ngpu = ts3_config.ntask
nnode = ts3_config.nnode
npernode = ngpu // nnode
logger.info(f"Using {ngpu} GPUs on {nnode} nodes, {npernode} per node.")
if ngpu == 1:
cmd_str = (
f". {ts3_config.environment_script};"
"turbostream input.hdf5 output 1 > log.txt"
)
else:
cmd_str = (
f". {ts3_config.environment_script};"
f" mpirun -npernode {npernode} -np {ngpu} turbostream"
f" input.hdf5 output {npernode} > log.txt"
)
# Remove old probe data
probe_dat = glob("output_probe_*.dat")
for fname in probe_dat:
os.remove(fname)
# Start the Turbostream process
with subprocess.Popen(
cmd_str, shell=True, stderr=subprocess.PIPE, preexec_fn=os.setsid
) as proc:
# Until process has finished, check regularly for divergence
try:
while proc.poll() is None:
timeout = 60
start = timer()
while (timer() - start) < timeout:
sleep(10)
if os.path.isfile("log.txt"):
break
if not os.path.isfile("log.txt"):
raise Exception(
f"Timed out after {timeout}s waiting for TS3 log file to appear"
)
if istep_nan := _check_nan("log.txt"):
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
raise ConvergenceError(
f"TS3 diverged at step {istep_nan}"
) from None
except KeyboardInterrupt:
logger.iter("******")
logger.iter("Caught interrupt, killing solver...")
logger.iter("******")
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
proc.wait()
logger.iter("Killed solver.")
proc.wait()
# If we have an error code, prind debugging info
if proc.returncode:
raise Exception(
f"""TS3 failed, exit code {proc.returncode}
COMMAND: {cmd_str}
STDERR: {proc.stderr.read().decode(sys.getfilesystemencoding()).strip()}
Are you on a HPC compute node, i.e. gpu-q-x not login-q-x?"""
) from None
# Delete extraneous files
for f in ("stopit", "output_avg.xdmf", "output.xdmf"):
try:
os.remove(f)
except FileNotFoundError:
pass
# Remove empty hdf5 probes (we don't use them)
probe_hdf5 = glob("output_probe_*.hdf5")
for fname in probe_hdf5:
os.remove(fname)
os.chdir(old_workdir)
def _read_hdf5(grid, ts3_config):
"""Using a given configuration, load flow solution and insert into grid."""
output_file_path = os.path.join(ts3_config.workdir, "output_avg.hdf5")
output_inst_file_path = os.path.join(ts3_config.workdir, "output.hdf5")
if not os.path.exists(output_file_path):
raise Exception(f"""No Turbostream output file found at: {output_file_path}""")
f = h5py.File(output_file_path, "r")
fi = h5py.File(output_inst_file_path, "r")
# Although the TS3 hdf5 reports the shape of the data as ni x nj x nk, this
# is not correct and actually the underlying data is stored in nk x nj x ni order.
# So we reshape and swap the axes back
def _unflip(x):
ni, nj, nk = x.shape
return np.swapaxes(np.reshape(x, (nk, nj, ni)), 0, 2)
# Loop over blocks
nb = len(grid)
for ib in range(nb):
block = grid[ib]
block_group = f[f"block{ib}"]
# Pull some properties first
rho = _unflip(block_group["ro_bp"])
roe = _unflip(block_group["roe_bp"])
trans_dyn_vis = _unflip(fi[f"block{ib}"]["trans_dyn_vis_bp"])
# Check for divergence
if not np.isfinite(rho).all():
raise ConvergenceError("TS3 solution has NAN density.")
if (rho < 0.0).any():
raise ConvergenceError("TS3 solution has negative density.")
if not np.isfinite(roe).all():
raise ConvergenceError("TS3 solution has NAN total energy.")
if (roe < 0.0).any():
raise ConvergenceError("TS3 solution has negative total energy.")
if not np.isfinite(trans_dyn_vis).all():
raise ConvergenceError("TS3 solution has NAN turbulent viscosity.")
# Set the velocities
block.Vx = _unflip(block_group["rovx_bp"]) / rho
block.Vr = _unflip(block_group["rovr_bp"]) / rho
block.Vt = _unflip(block_group["rorvt_bp"]) / rho / block.r
# Convert total energy to internal energy
u = roe / rho - 0.5 * block.V**2.0
# Make sure that u is positive
if not (u > 0.0).all():
raise ConvergenceError("TS3 solution has negative internal energy.")
# Set the thermodynamic state
Tu0_old = block.Tu0 + 0.0
block.Tu0 = 0.0
block.set_rho_u(rho, u)
block.set_Tu0(Tu0_old)
# Set turbulent viscosity
block.mu_turb = trans_dyn_vis
# Print yplus if requested
if ts3_config.show_yplus:
yplus = _unflip(block_group["yplus_bp"])
# Remove not-wall nodes
yplus = yplus[yplus > 0.0]
logger.info(f"Block {ib} ({block.label}): mean yplus={yplus.mean():.1f}")
f.close()
fi.close()
def _run(grid, ts3_config):
"""Perform all steps on a grid and config."""
_write_hdf5(grid, ts3_config)
_execute(ts3_config)
_read_hdf5(grid, ts3_config)
def run(grid, ts3_conf, machine):
"""Write, run, and read TS3 results for a grid object, specifying some settings.
Parameters
----------
grid
ts3_conf
machine
"""
# Check that the user is a member of the turbostream group
try:
ts_users = grp.getgrnam("turbostream").gr_mem
current_user = getpass.getuser()
if current_user not in ts_users:
raise Exception(
f"Current user {current_user} is not a member of the turbostream group"
)
except KeyError:
if not ts3_conf.skip:
raise Exception("Cannot locate turbostream - are you on the HPC?") from None
# Make workdir if needed
if not os.path.exists(ts3_conf.workdir):
os.makedirs(ts3_conf.workdir)
input_file_path = os.path.join(ts3_conf.workdir, "input.hdf5")
output_file_path = os.path.join(ts3_conf.workdir, "output_avg.hdf5")
output_inst_file_path = os.path.join(ts3_conf.workdir, "output.hdf5")
soln_exists = os.path.exists(output_file_path) and os.path.exists(
output_inst_file_path
)
if ts3_conf.skip and soln_exists:
logger.info("Skipping running, loading previous solution.")
try:
_read_hdf5(grid, ts3_conf)
except ValueError:
logger.info("Failed, will continue with initial guess.")
return
# Final check of the mesh
grid.match_patches()
for block in grid:
# block.check_coordinates()
block.check_wall_distance()
# Load balancing
try:
ts3_conf.ntask = int(np.minimum(int(os.environ["SLURM_NTASKS"]), len(grid)))
ts3_conf.nnode = int(os.environ["SLURM_NNODES"])
except KeyError:
ts3_conf.ntask = 1
ts3_conf.nnode = 1
logger.info(
"Could not establish number of GPUs, assuming serial "
"(are you on a compute node?)"
)
if ts3_conf.skip:
logger.info("Skipping running, reloading initial guess.")
_write_hdf5(grid, ts3_conf)
shutil.copy(input_file_path, output_file_path)
shutil.copy(input_file_path, output_inst_file_path)
_read_hdf5(grid, ts3_conf)
return
_run(grid, ts3_conf)
# Produce a warning if the outlet is choked
grid.check_outlet_choke()
# Parse the log file
log_path = os.path.join(ts3_conf.workdir, "log.txt")
state_log = grid.inlet_patches[0].state.copy()
state_log.set_Tu0(0.0)
istep_save_start = ts3_conf.application_variables(0.0, 0.0, 0.0)["nstep_save_start"]
return ConvergenceHistory(*parse_log(log_path), state_log, istep_save_start)
re_nstep = re.compile(r"nstep\s*:\s*(\d*)$")
re_cp = re.compile(r"cp\s*:\s*(\d*\.\d*)$")
re_dts = re.compile(r"dts\s*:\s*(\d*)$")
re_ncycle = re.compile(r"ncycle\s*:\s*(\d*)$")
re_davg = re.compile(r"TOTAL DAVG \s*(\d*\.\d*)E([+-]\d*)")
re_nstep_cycle = re.compile(r"nstep_cycle\s*:\s*(\d*)$")
re_nstep_save_start = re.compile(r"nstep_save_start\s*:\s*(\d*)$")
re_mdot = re.compile(r"^INLET FLOW =\s*(-?\d*\.\d*)\s*OUTLET FLOW =\s*(-?\d*\.\d*)$")
re_Po = re.compile(
r"^AVG INLET STAG P =\s*(-?\d*\.\d*)\s*AVG OUTLET STAG P =\s*(-?\d*\.\d*)$"
)
re_To = re.compile(
r"^AVG INLET STAG T =\s*(-?\d*\.\d*)\s*AVG OUTLET STAG T =\s*(-?\d*\.\d*)$"
)
re_eta = re.compile(r"EFFICIENCY\s*=\s*(-?\d*.\d*)$")
re_nan = re.compile(r".*NAN.*")
re_current_step = re.compile(r"^O?U?T?E?R? ?STEP No\.\s*(\d*)", flags=re.MULTILINE)
def parse_log(fname):
"""Read residuals and boundary properties from log file.
Parameters
----------
fname: string
File name of a Turbostream 3 log.
Returns
-------
istep: (nlog) array
mdot: (2, nlog) array
ho: (2, nlog) array
Po: (2, nlog) array
resid: (nlog) array
"""
logger.debug(f"Opening log file {fname}...")
# Loop over lines in the file
with open(fname, "r") as f:
# Look for cp
for line in f:
match = re_cp.search(line)
if match:
cp = float(match.group(1))
break
# Look for number of steps
for line in f:
match = re_nstep.search(line)
if match:
nstep = int(match.group(1))
break
# Look for number of steps
for line in f:
match = re_dts.search(line)
if match:
dts = int(match.group(1))
break
# Look for averaging steps
for line in f:
match = re_nstep_save_start.search(line)
if match:
# nstep_save_start = int(match.group(1))
break
# Preallocate
step_now = 0
dn = 1 if dts else 50
nlog = nstep // dn
istep = np.arange(nlog) * dn
mdot = np.zeros((2, nlog))
Po = np.zeros((2, nlog))
To = np.zeros((2, nlog))
resid = np.zeros((nlog,))
for ilog in range(nlog):
logger.debug(f"* Parsing istep={istep[ilog]}")
# Look for residual
if ilog > 0:
for line in f:
if davg_match := re_davg.search(line):
logger.debug(f'Found: "{line.strip()}"')
sig = float(davg_match.group(1))
expon = int(davg_match.group(2))
resid[ilog] = sig * 10 ** (expon)
break
else:
resid[ilog] = np.nan
try:
if not dts:
# Loop over lines until we find mdot
logger.debug("Finding mass flow rate...")
for line in f:
if mdot_match := re_mdot.search(line):
logger.debug(f'Found: "{line.strip()}"')
mdot[:, ilog] = [float(m) for m in mdot_match.group(1, 2)]
break
else:
for line in f:
if re_nstep.search(line):
logger.debug(f'Found: "{line.strip()}"')
break
# Skip flow ratio
_ = f.readline()
# Stagnation pressures
ln = f.readline()
logger.debug(f'Reading Po from "{ln.strip()}"')
match_Po = re_Po.search(ln)
Po[:, ilog] = [float(m) for m in match_Po.group(1, 2)]
# Stagnation temperatures
ln = f.readline()
logger.debug(f'Reading To from "{ln.strip()}"')
match_To = re_To.search(ln)
To[:, ilog] = [float(m) for m in match_To.group(1, 2)]
# Skip power and effy
_ = f.readline()
_ = f.readline()
_ = f.readline()
# Next step number
if ilog < nlog - 1:
logger.debug("Finding next step No...")
step_next = None
for line in f:
if step_match := re_current_step.search(line):
step_next = int(step_match.group(1))
if step_next > step_now:
logger.debug(f" Found next istep={step_next}")
step_now = step_next
break
else:
continue
if not step_next == istep[ilog + 1]:
raise Exception(f"Log step mismatch at {step_now}, {step_next}")
except AttributeError:
logger.debug("Failed to parse, breaking")
break
return istep, mdot, To * cp, Po, resid
@property
def nstep(self):
return np.linspace(0, self._nlog - 1, self._nlog, dtype=int) * self._step_fac
def __str__(self):
return (
f"TS3Log(mdot_drift={self.mdot_drift*100.:.2f}%,"
f"mdot_err={self.mdot_err*100:.2f}%,"
f"eta_drift={self.eta_drift*100:.2f}℅"
)
@property
def eta_drift(self):
eta_avg = self._eta[self.nstep >= self._nstep_save_start]
# Consider instantaneous eta
# drift = eta_avg - eta_avg[-1]
# return drift[np.argmax(np.abs(drift))]
# Split averaging period into two and check eta same for both
n2 = len(eta_avg) // 2
return eta_avg[:n2].mean() - eta_avg[n2:].mean()
@property
def mdot_err(self):
err = (self._mdot[0] / self._mdot[1] - 1.0)[
self.nstep >= self._nstep_save_start
]
return err[np.argmax(np.abs(err))]
@property
def mdot_drift(self):
drift = self._mdot[0] / self._mdot[0, -1] - 1.0
err = drift[self.nstep >= self._nstep_save_start]
return err[np.argmax(np.abs(err))]
def read_probe_dat_dir(dname, S, shape):
fnames = glob(os.path.join(dname, "*.dat"))
return [read_probe_dat(f, S, shape) for f in fnames]
def read_probe_dat(fname, S, shape=()):
logger.info(f"Reading {fname}")
Npts = int(np.loadtxt(fname, max_rows=1))
x, r, rt, ro, rovx, rovr, rorvt, roe = np.loadtxt(fname, skiprows=1).T
nt = len(x) // Npts
Fshape = shape + (nt,)
if shape:
x = x.reshape(Fshape, order="F")
r = r.reshape(Fshape, order="F")
rt = rt.reshape(Fshape, order="F")
ro = ro.reshape(Fshape, order="F")
rovx = rovx.reshape(Fshape, order="F")
rovr = rovr.reshape(Fshape, order="F")
rorvt = rorvt.reshape(Fshape, order="F")
roe = roe.reshape(Fshape, order="F")
Fshape = x.shape
F = turbigen.flowfield.PerfectFlowField(Fshape)
F.Tu0 = 0.0
F.cp = S.cp
F.gamma = S.gamma
F.mu = S.mu
F.Omega = 0.0
F.xrt = np.stack((x, r, rt / r))
F.Vxrt = np.stack((rovx, rovr, rorvt / r)) / ro
u = roe / ro - 0.5 * F.V**2.0
F.set_rho_u(ro, u)
F.set_Tu0(S.Tu0)
return F
def _check_nan(fname):
"""Return step number of divergence from TS3 log, or zero if no NANs found."""
NBYTES = 1024
with open(fname, "r") as f:
while chunk := f.read(NBYTES):
if re_nan.match(chunk):
try:
return int(re_current_step.findall(chunk)[-1])
except Exception:
return -1
return 0
def _get_time_vector(ts3_config):
freq = ts3_config.frequency
nstep_cycle = ts3_config.nstep_cycle
nt = nstep_cycle * ts3_config.ncycle
it = np.arange(nt)
dt = 1.0 / freq / nstep_cycle
t = it * dt
return t