Skip to content

Commit

Permalink
Manager persistence updates (standardised storage volume and Flyway e…
Browse files Browse the repository at this point in the history 
…nv variable) (openremote#1004)
  • Loading branch information
@richturner
richturner authored Mar 16, 2023
1 parent 43e0c99 commit b217d55
Show file tree
Hide file tree
Showing 23 changed files with 204 additions and 109 deletions.
25 changes: 18 additions & 7 deletions agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.openremote.agent.protocol.AbstractProtocol;
import org.openremote.agent.protocol.bluetooth.mesh.models.SigModelParser;
import org.openremote.agent.protocol.bluetooth.mesh.utils.MeshParserUtils;
import org.openremote.container.persistence.PersistenceService;
import org.openremote.model.Container;
import org.openremote.model.asset.Asset;
import org.openremote.model.asset.AssetTreeNode;
Expand All @@ -43,6 +44,7 @@
import org.openremote.model.value.ValueFormat;
import org.openremote.model.value.ValueType;

import java.nio.file.Path;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -74,9 +76,9 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,

public static final Logger LOG = SyslogCategory.getLogger(SyslogCategory.PROTOCOL, BluetoothMeshProtocol.class.getName());

private static MainThreadManager mainThread = new MainThreadManager();
private static final MainThreadManager mainThread = new MainThreadManager();
private static ScheduledFuture<?> mainThreadFuture = null;
private static BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {
private static final BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {
@Override
public void onConnectedPeripheral(BluetoothPeripheral peripheral) {
LOG.info("BluetoothCentralManager::onConnectedPeripheral: [Name=" + peripheral.getName() + ", Address=" + peripheral.getAddress() + "]");
Expand Down Expand Up @@ -132,10 +134,10 @@ public void onScanFailed(int errorCode) {
}
}
};
private static BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);
private static List<BluetoothMeshNetwork> networkList = new LinkedList<>();
private static SequenceNumberPersistencyManager sequenceNumberManager = new SequenceNumberPersistencyManager();

private static final BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);
private static final List<BluetoothMeshNetwork> networkList = new LinkedList<>();
// Not ideal this but will do for now
private static SequenceNumberPersistencyManager sequenceNumberManager;

public synchronized static void initMainThread(ScheduledExecutorService executorService) {
if (mainThreadFuture == null) {
Expand Down Expand Up @@ -180,6 +182,15 @@ public String getProtocolInstanceUri() {

@Override
protected synchronized void doStart(Container container) throws Exception {

synchronized (BluetoothMeshProtocol.mainThread) {
if (BluetoothMeshProtocol.sequenceNumberManager == null) {
Path storagePath = container.getService(PersistenceService.class).getStorageDir();
BluetoothMeshProtocol.sequenceNumberManager = new SequenceNumberPersistencyManager(storagePath.resolve("bluetoothmesh"));
BluetoothMeshProtocol.sequenceNumberManager.load();
}
}

LOG.info("Starting Bluetooth Mesh protocol.");
String meshNetKeyParam = agent.getNetworkKey().orElseThrow(() -> {
String msg = "No Bluetooth Mesh network key provided for protocol: " + this;
Expand Down Expand Up @@ -243,7 +254,7 @@ public void accept(ConnectionStatus connectionStatus) {
}
}
};
BluetoothMeshProtocol.sequenceNumberManager.load();

Integer oldSequenceNumber = BluetoothMeshProtocol.sequenceNumberManager.getSequenceNumber(networkKey, sourceAddress);
if (oldSequenceNumber == null) {
oldSequenceNumber = sequenceNumberParam;
Expand Down
41 changes: 17 additions & 24 deletions ...n/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

public class SequenceNumberPersistencyManager {
Expand All @@ -48,7 +49,11 @@ public class SequenceNumberPersistencyManager {
public static String XML_TAG_SEQUENCE_NUMBER = "SequenceNumber";

private final Map<String, Map<String, Integer>> map = new HashMap<>();
private final URI filePath = new File("/btmesh/sequencenumber.xml").toURI();
private final Path filePath;

public SequenceNumberPersistencyManager(Path storagePath) {
filePath = storagePath.resolve("sequencenumber.xml");
}

public synchronized void save(NetworkKey networkKey, int unicastAddress, int sequenceNumber) {
String key = MeshParserUtils.bytesToHex(networkKey.key, false);
Expand All @@ -65,7 +70,7 @@ public synchronized void save(NetworkKey networkKey, int unicastAddress, int seq
save();
}

public synchronized void save() {
protected synchronized void save() {
Document doc = new Document();

Element root = new Element(XML_TAG_SEQUENCE_NUMBER_LIST);
Expand Down Expand Up @@ -96,8 +101,8 @@ public synchronized void save() {
FileWriter writer = null;
File file = null;
try {
file = new File(filePath);
writer = new FileWriter(new File(filePath));
file = filePath.toFile();
writer = new FileWriter(file);
outputter.output(doc, writer);
writer.flush();
} catch (IOException e) {
Expand All @@ -114,21 +119,16 @@ public synchronized void save() {
}

public synchronized void load() {
if (!fileExists()) {
if (!filePath.toFile().exists()) {
return;
}
File file = new File(filePath);
File file = filePath.toFile();
SAXBuilder builder = new SAXBuilder();
Document doc = null;
Document doc;
try {
doc = builder.build(file);
} catch (JDOMException e) {
LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
e.printStackTrace();
return;
} catch (IOException e) {
LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
e.printStackTrace();
} catch (JDOMException | IOException e) {
LOG.log(Level.SEVERE, "Failed to load sequence number file: '" + file.getPath() + "'", e);
return;
}
map.clear();
Expand All @@ -147,14 +147,13 @@ public synchronized void load() {
String number = numberElement.getText();

if (!map.containsKey(key)) {
map.put(key, new HashMap<String, Integer>());
map.put(key, new HashMap<>());
}
try {
map.get(key).put(address, Integer.valueOf(number));
} catch (NumberFormatException e) {
LOG.severe("Error while loading sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
LOG.log(Level.SEVERE, "Error while loading sequence number file: '" + file.getPath() + "'", e);
e.printStackTrace();
continue;
}
}
}
Expand All @@ -168,10 +167,4 @@ public synchronized Integer getSequenceNumber(NetworkKey networkKey, int unicast
}
return number;
}

// Private Instance Methods -------------------------------------------------------------------

private boolean fileExists() {
return new File(filePath).exists();
}
}
6 changes: 6 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,9 @@ task checkFilesGitIgnoredNew(type: Exec) {

commandLine args
}

task clean() {
doLast {
delete "tmp"
}
}
37 changes: 35 additions & 2 deletions container/src/main/java/org/openremote/container/persistence/PersistenceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,13 @@
import javax.persistence.spi.PersistenceUnitTransactionType;
import javax.sql.DataSource;
import javax.ws.rs.core.UriBuilder;
import java.io.File;
import java.lang.reflect.Field;
import java.net.URL;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
Expand Down Expand Up @@ -225,6 +230,9 @@ public ClassLoader getNewTempClassLoader() {
public static final int OR_DB_MAX_POOL_SIZE_DEFAULT = 20;
public static final String OR_DB_CONNECTION_TIMEOUT_SECONDS = "OR_DB_CONNECTION_TIMEOUT_SECONDS";
public static final int OR_DB_CONNECTION_TIMEOUT_SECONDS_DEFAULT = 300;
public static final String OR_STORAGE_DIR = "OR_STORAGE_DIR";
public static final String OR_STORAGE_DIR_DEFAULT = "tmp";
public static final String OR_DB_FLYWAY_OUT_OF_ORDER = "OR_DB_FLYWAY_OUT_OF_ORDER";
public static final int PRIORITY = Integer.MIN_VALUE + 100;

protected MessageBrokerService messageBrokerService;
Expand All @@ -236,6 +244,7 @@ public ClassLoader getNewTempClassLoader() {
protected boolean forceClean;
protected Set<String> defaultSchemaLocations = new HashSet<>();
protected Set<String> schemas = new HashSet<>();
protected Path storageDir;

public static Predicate isPersistenceEventForEntityType(Class<?> type) {
return exchange -> {
Expand Down Expand Up @@ -292,8 +301,24 @@ public void init(Container container) throws Exception {

forceClean = getBoolean(container.getConfig(), OR_SETUP_RUN_ON_RESTART, container.isDevMode());

storageDir = Paths.get(getString(container.getConfig(), OR_STORAGE_DIR, OR_STORAGE_DIR_DEFAULT));
LOG.log(Level.INFO, "Setting storage directory to '" + storageDir.toAbsolutePath() + "'");

if (!Files.exists(storageDir) || !Files.isDirectory(storageDir)) {
String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' doesn't exist or is not a folder";
LOG.log(Level.SEVERE, msg);
throw new FileSystemNotFoundException(msg);
} else {
File testFile = storageDir.toFile();
if (!testFile.canRead() || !testFile.canWrite()) {
String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' is not writable";
LOG.log(Level.SEVERE, msg);
throw new FileSystemNotFoundException(msg);
}
}

openDatabase(container, database, dbUsername, dbPassword, connectionUrl);
prepareSchema(connectionUrl, dbUsername, dbPassword, dbSchema);
prepareSchema(container, connectionUrl, dbUsername, dbPassword, dbSchema);

// Register standard entity classes and also any Entity ClassProviders
List<String> entityClasses = new ArrayList<>(50);
Expand Down Expand Up @@ -480,7 +505,10 @@ protected void openDatabase(Container container, Database database, String usern
database.open(persistenceUnitProperties, connectionUrl, username, password, connectionTimeoutSeconds, databaseMinPoolSize, databaseMaxPoolSize);
}

protected void prepareSchema(String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {
protected void prepareSchema(Container container, String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {

boolean outOfOrder = getBoolean(container.getConfig(), OR_DB_FLYWAY_OUT_OF_ORDER, false);

LOG.fine("Preparing database schema");
List<String> locations = new ArrayList<>();
List<String> schemas = new ArrayList<>();
Expand All @@ -503,6 +531,7 @@ protected void prepareSchema(String connectionUrl, String databaseUsername, Stri
.locations(locations.toArray(new String[0]))
.initSql(initSql.toString())
.baselineOnMigrate(true)
.outOfOrder(outOfOrder)
.load();

MigrationInfo currentMigration;
Expand Down Expand Up @@ -565,6 +594,10 @@ public String toString() {
'}';
}

public Path getStorageDir() {
return storageDir;
}

public static Field[] getEntityPropertyFields(Class<?> clazz, List<String> includeFields, List<String> excludeFields) {
return Arrays.stream(clazz.getDeclaredFields())
.filter(field -> ((field.isAnnotationPresent(Column.class) || field.isAnnotationPresent(EmbeddedId.class) || field.isAnnotationPresent(JoinColumn.class) || field.isAnnotationPresent(Formula.class)) && (excludeFields == null || !excludeFields.contains(field.getName())))
Expand Down
Empty file added deployment/manager/app/images/.appignore
View file
Empty file.
12 changes: 3 additions & 9 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ version: '2.4'

volumes:
proxy-data:
temp-data:
manager-data:
postgresql-data:
# btmesh-data:

services:

Expand Down Expand Up @@ -39,7 +38,7 @@ services:
image: openremote/postgresql:${POSTGRESQL_VERSION:-latest}
volumes:
- postgresql-data:/var/lib/postgresql/data
- temp-data:/tmp
- manager-data:/storage

keycloak:
restart: always
Expand Down Expand Up @@ -83,9 +82,4 @@ services:
OR_SETUP_IMPORT_DEMO_AGENT_KNX:
OR_SETUP_IMPORT_DEMO_AGENT_VELBUS:
volumes:
- temp-data:/tmp
# - /var/run/dbus:/var/run/dbus
# # Bluetooth mesh volume
# - btmesh-data:/btmesh
# devices:
# - /dev/ttyACM0:/dev/ttyS0
- manager-data:/storage
3 changes: 2 additions & 1 deletion manager/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ ENV OR_DB_PASSWORD ${OR_DB_PASSWORD:-postgres}
ENV OR_DB_MIN_POOL_SIZE ${OR_DB_MIN_POOL_SIZE:-5}
ENV OR_DB_MAX_POOL_SIZE ${OR_DB_MAX_POOL_SIZE:-20}
ENV OR_DB_CONNECTION_TIMEOUT_SECONDS ${OR_DB_CONNECTION_TIMEOUT_SECONDS:-300}
ENV OR_DB_FLYWAY_OUT_OF_ORDER ${OR_DB_FLYWAY_OUT_OF_ORDER}
ENV OR_KEYCLOAK_HOST ${OR_KEYCLOAK_HOST:-keycloak}
ENV OR_KEYCLOAK_PORT ${OR_KEYCLOAK_PORT:-8080}
ENV OR_KEYCLOAK_GRANT_FILE ${OR_KEYCLOAK_GRANT_FILE:-/deployment/manager/keycloak.json}
ENV OR_APP_DOCROOT ${OR_APP_DOCROOT:-/opt/web}
ENV OR_CUSTOM_APP_DOCROOT ${OR_CUSTOM_APP_DOCROOT:-/deployment/manager/app}
ENV OR_PROVISIONING_DOCROOT ${OR_PROVISIONING_DOCROOT:-/deployment/manager/provisioning}
ENV OR_ROOT_REDIRECT_PATH ${OR_ROOT_REDIRECT_PATH:-/manager}
ENV OR_DATA_POINTS_EXPORT_DIR ${OR_DATA_POINTS_EXPORT_DIR:-/tmp}
ENV OR_LOGGING_CONFIG_FILE ${OR_LOGGING_CONFIG_FILE}
ENV OR_MAP_TILES_PATH ${OR_MAP_TILES_PATH:-/deployment.local/mapdata/mapdata.mbtiles}
ENV OR_MAP_SETTINGS_PATH ${OR_MAP_SETTINGS_PATH:-/deployment/map/mapsettings.json}
Expand All @@ -54,6 +54,7 @@ ENV OR_RULE_EVENT_EXPIRES ${OR_RULE_EVENT_EXPIRES:-PT1H}
ENV OR_IDENTITY_PROVIDER ${OR_IDENTITY_PROVIDER:-keycloak}
ENV OR_IDENTITY_SESSION_MAX_MINUTES ${OR_IDENTITY_SESSION_MAX_MINUTES:-1440}
ENV OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES ${OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES:-2628000}
ENV OR_STORAGE_DIR ${OR_STORAGE_DIR:-/storage}
ENV OR_JAVA_OPTS ${OR_JAVA_OPTS:--Xms500m -Xmx2g \
-XX:NativeMemoryTracking=summary \
-Xlog:all=warning:stdout:uptime,level,tags \
Expand Down
Loading

0 comments on commit b217d55

Please sign in to comment.