Skip to content

Commit

Permalink
Fixes and enhancements for add-on services (#3293)
Browse files Browse the repository at this point in the history 
* Add an add-on service that provides add-ons in the addons-folder
* fix connection and countries

This is needed to show the add-ons from the addons in the UI.

Signed-off-by: Jan N. Klug <[email protected]>
  • Loading branch information
@J-N-K
J-N-K authored Jan 21, 2023
1 parent 1c34321 commit 6738277
Show file tree
Hide file tree
Showing 22 changed files with 369 additions and 106 deletions.
6 changes: 3 additions & 3 deletions ...g/openhab/core/addon/marketplace/karaf/internal/community/CommunityKarafAddonHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@ public boolean isInstalled(String addonId) {
public void install(Addon addon) throws MarketplaceHandlerException {
try {
URL sourceUrl = new URL((String) addon.getProperties().get(KAR_DOWNLOAD_URL_PROPERTY));
addKarToCache(addon.getId(), sourceUrl);
installFromCache(addon.getId());
addKarToCache(addon.getUid(), sourceUrl);
installFromCache(addon.getUid());
} catch (MalformedURLException e) {
throw new MarketplaceHandlerException("Malformed source URL: " + e.getMessage(), e);
}
Expand All @@ -116,7 +116,7 @@ public void install(Addon addon) throws MarketplaceHandlerException {
@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
try {
Path addonPath = getAddonCacheDirectory(addon.getId());
Path addonPath = getAddonCacheDirectory(addon.getUid());
List<String> repositories = karService.list();
for (Path path : karFilesStream(addonPath).collect(Collectors.toList())) {
String karRepoName = pathToKarRepoName(path);
Expand Down
23 changes: 12 additions & 11 deletions ...ketplace/src/main/java/org/openhab/core/addon/marketplace/AbstractRemoteAddonService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,15 +104,16 @@ public void refreshSource() {
.forEach(addons::add);

// create lookup list to make sure installed addons take precedence
List<String> installedAddons = addons.stream().map(Addon::getId).collect(Collectors.toList());
List<String> installedAddons = addons.stream().map(Addon::getUid).collect(Collectors.toList());

if (remoteEnabled()) {
List<Addon> remoteAddons = Objects.requireNonNullElse(cachedRemoteAddons.getValue(), List.of());
remoteAddons.stream().filter(a -> !installedAddons.contains(a.getId())).forEach(addons::add);
remoteAddons.stream().filter(a -> !installedAddons.contains(a.getUid())).forEach(addons::add);
}

// check real installation status based on handlers
addons.forEach(addon -> addon.setInstalled(addonHandlers.stream().anyMatch(h -> h.isInstalled(addon.getId()))));
addons.forEach(
addon -> addon.setInstalled(addonHandlers.stream().anyMatch(h -> h.isInstalled(addon.getUid()))));

// remove incompatible add-ons if not enabled
boolean showIncompatible = includeIncompatible();
Expand Down Expand Up @@ -171,17 +172,17 @@ public void install(String id) {
if (addon != null) {
for (MarketplaceAddonHandler handler : addonHandlers) {
if (handler.supports(addon.getType(), addon.getContentType())) {
if (!handler.isInstalled(addon.getId())) {
if (!handler.isInstalled(addon.getUid())) {
try {
handler.install(addon);
installedAddonStorage.put(id, gson.toJson(addon));
refreshSource();
postInstalledEvent(addon.getId());
postInstalledEvent(addon.getUid());
} catch (MarketplaceHandlerException e) {
postFailureEvent(addon.getId(), e.getMessage());
postFailureEvent(addon.getUid(), e.getMessage());
}
} else {
postFailureEvent(addon.getId(), "Add-on is already installed.");
postFailureEvent(addon.getUid(), "Add-on is already installed.");
}
return;
}
Expand All @@ -196,18 +197,18 @@ public void uninstall(String id) {
if (addon != null) {
for (MarketplaceAddonHandler handler : addonHandlers) {
if (handler.supports(addon.getType(), addon.getContentType())) {
if (handler.isInstalled(addon.getId())) {
if (handler.isInstalled(addon.getUid())) {
try {
handler.uninstall(addon);
installedAddonStorage.remove(id);
refreshSource();
postUninstalledEvent(addon.getId());
postUninstalledEvent(addon.getUid());
} catch (MarketplaceHandlerException e) {
postFailureEvent(addon.getId(), e.getMessage());
postFailureEvent(addon.getUid(), e.getMessage());
}
} else {
installedAddonStorage.remove(id);
postFailureEvent(addon.getId(), "Add-on is not installed.");
postFailureEvent(addon.getUid(), "Add-on is not installed.");
}
return;
}
Expand Down
6 changes: 3 additions & 3 deletions ...g/openhab/core/addon/marketplace/internal/community/CommunityBlockLibaryAddonHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ public void install(Addon addon) throws MarketplaceHandlerException {
String yamlContent = (String) addon.getProperties().get(YAML_CONTENT_PROPERTY);

if (yamlDownloadUrl != null) {
addWidgetAsYAML(addon.getId(), getWidgetFromURL(yamlDownloadUrl));
addWidgetAsYAML(addon.getUid(), getWidgetFromURL(yamlDownloadUrl));
} else if (yamlContent != null) {
addWidgetAsYAML(addon.getId(), yamlContent);
addWidgetAsYAML(addon.getUid(), yamlContent);
} else {
throw new IllegalArgumentException("Couldn't find the block library in the add-on entry");
}
Expand All @@ -96,7 +96,7 @@ public void install(Addon addon) throws MarketplaceHandlerException {

@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
blocksRegistry.getAll().stream().filter(w -> w.hasTag(addon.getId())).forEach(w -> {
blocksRegistry.getAll().stream().filter(w -> w.hasTag(addon.getUid())).forEach(w -> {
blocksRegistry.remove(w.getUID());
});
}
Expand Down
6 changes: 3 additions & 3 deletions ...va/org/openhab/core/addon/marketplace/internal/community/CommunityBundleAddonHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,16 +75,16 @@ public boolean isInstalled(String id) {
public void install(Addon addon) throws MarketplaceHandlerException {
try {
URL sourceUrl = new URL((String) addon.getProperties().get(JAR_DOWNLOAD_URL_PROPERTY));
addBundleToCache(addon.getId(), sourceUrl);
installFromCache(bundleContext, addon.getId());
addBundleToCache(addon.getUid(), sourceUrl);
installFromCache(bundleContext, addon.getUid());
} catch (MalformedURLException e) {
throw new MarketplaceHandlerException("Malformed source URL: " + e.getMessage(), e);
}
}

@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
uninstallBundle(bundleContext, addon.getId());
uninstallBundle(bundleContext, addon.getUid());
}

@Override
Expand Down
43 changes: 32 additions & 11 deletions ...g/openhab/core/addon/marketplace/internal/community/CommunityMarketplaceAddonService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -88,6 +89,7 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
private static final String COMMUNITY_BASE_URL = "https://community.openhab.org";
private static final String COMMUNITY_MARKETPLACE_URL = COMMUNITY_BASE_URL + "/c/marketplace/69/l/latest";
private static final String COMMUNITY_TOPIC_URL = COMMUNITY_BASE_URL + "/t/";
private static final Pattern BUNDLE_NAME_PATTERN = Pattern.compile(".*/(.*)-\\d+\\.\\d+\\.\\d+.*");

private static final String SERVICE_ID = "marketplace";
private static final String ADDON_ID_PREFIX = SERVICE_ID + ":";
Expand Down Expand Up @@ -195,12 +197,11 @@ protected List<Addon> getRemoteAddons() {
}

@Override
public @Nullable Addon getAddon(String id, @Nullable Locale locale) {
String fullId = ADDON_ID_PREFIX + id;
public @Nullable Addon getAddon(String uid, @Nullable Locale locale) {
// check if it is an installed add-on (cachedAddons also contains possibly incomplete results from the remote
// side, we need to retrieve them from Discourse)
if (installedAddons.contains(fullId)) {
return cachedAddons.stream().filter(e -> fullId.equals(e.getId())).findAny().orElse(null);
if (installedAddons.contains(uid)) {
return cachedAddons.stream().filter(e -> uid.equals(e.getUid())).findAny().orElse(null);
}

if (!remoteEnabled()) {
Expand All @@ -209,7 +210,7 @@ protected List<Addon> getRemoteAddons() {

// retrieve from remote
try {
URL url = new URL(String.format("%s%s", COMMUNITY_TOPIC_URL, id.replace(ADDON_ID_PREFIX, "")));
URL url = new URL(String.format("%s%s", COMMUNITY_TOPIC_URL, uid.replace(ADDON_ID_PREFIX, "")));
URLConnection connection = url.openConnection();
connection.addRequestProperty("Accept", "application/json");
if (this.apiKey != null) {
Expand Down Expand Up @@ -280,9 +281,10 @@ private String getContentType(@Nullable Integer category, List<String> tags) {
private Addon convertTopicItemToAddon(DiscourseTopicItem topic, List<DiscourseUser> users) {
List<String> tags = Arrays.asList(Objects.requireNonNullElse(topic.tags, new String[0]));

String id = ADDON_ID_PREFIX + topic.id.toString();
String uid = ADDON_ID_PREFIX + topic.id.toString();
AddonType addonType = getAddonType(topic.categoryId, tags);
String type = (addonType != null) ? addonType.getId() : "";
String id = topic.id.toString(); // this will be replaced after installation by the correct id if available
String contentType = getContentType(topic.categoryId, tags);

String title = topic.title;
Expand Down Expand Up @@ -329,9 +331,9 @@ private Addon convertTopicItemToAddon(DiscourseTopicItem topic, List<DiscourseUs

// try to use a handler to determine if the add-on is installed
boolean installed = addonHandlers.stream()
.anyMatch(handler -> handler.supports(type, contentType) && handler.isInstalled(id));
.anyMatch(handler -> handler.supports(type, contentType) && handler.isInstalled(uid));

return Addon.create(id).withType(type).withContentType(contentType).withImageLink(topic.imageUrl)
return Addon.create(uid).withType(type).withId(id).withContentType(contentType).withImageLink(topic.imageUrl)
.withAuthor(author).withProperties(properties).withLabel(title).withInstalled(installed)
.withMaturity(maturity).withCompatible(compatible).withLink(link).build();
}
Expand All @@ -354,7 +356,7 @@ private String unescapeEntities(String content) {
* @return the list item
*/
private Addon convertTopicToAddon(DiscourseTopicResponseDTO topic) {
String id = ADDON_ID_PREFIX + topic.id.toString();
String uid = ADDON_ID_PREFIX + topic.id.toString();
List<String> tags = Arrays.asList(Objects.requireNonNullElse(topic.tags, new String[0]));

AddonType addonType = getAddonType(topic.categoryId, tags);
Expand All @@ -380,15 +382,18 @@ private Addon convertTopicToAddon(DiscourseTopicResponseDTO topic) {
properties.put("tags", tags.toArray(String[]::new));

String detailedDescription = topic.postStream.posts[0].cooked;
String id = null;

// try to extract contents or links
if (topic.postStream.posts[0].linkCounts != null) {
for (DiscoursePostLink postLink : topic.postStream.posts[0].linkCounts) {
if (postLink.url.endsWith(".jar")) {
properties.put("jar_download_url", postLink.url);
id = determineIdFromUrl(postLink.url);
}
if (postLink.url.endsWith(".kar")) {
properties.put("kar_download_url", postLink.url);
id = determineIdFromUrl(postLink.url);
}
if (postLink.url.endsWith(".json")) {
properties.put("json_download_url", postLink.url);
Expand All @@ -398,6 +403,11 @@ private Addon convertTopicToAddon(DiscourseTopicResponseDTO topic) {
}
}
}

if (id == null) {
id = topic.id.toString(); // this is a fallback if we couldn't find a better id
}

if (detailedDescription.contains(JSON_CODE_MARKUP_START)) {
String jsonContent = detailedDescription.substring(
detailedDescription.indexOf(JSON_CODE_MARKUP_START) + JSON_CODE_MARKUP_START.length(),
Expand All @@ -413,12 +423,23 @@ private Addon convertTopicToAddon(DiscourseTopicResponseDTO topic) {

// try to use a handler to determine if the add-on is installed
boolean installed = addonHandlers.stream()
.anyMatch(handler -> handler.supports(type, contentType) && handler.isInstalled(id));
.anyMatch(handler -> handler.supports(type, contentType) && handler.isInstalled(uid));

return Addon.create(id).withType(type).withContentType(contentType).withLabel(topic.title)
return Addon.create(uid).withType(type).withId(id).withContentType(contentType).withLabel(topic.title)
.withImageLink(topic.imageUrl).withLink(COMMUNITY_TOPIC_URL + topic.id.toString())
.withAuthor(topic.postStream.posts[0].displayUsername).withMaturity(maturity)
.withDetailedDescription(detailedDescription).withInstalled(installed).withProperties(properties)
.build();
}

private @Nullable String determineIdFromUrl(String url) {
Matcher matcher = BUNDLE_NAME_PATTERN.matcher(url);
if (matcher.matches()) {
String bundleName = matcher.group(1);
return bundleName.substring(bundleName.lastIndexOf(".") + 1);
} else {
logger.warn("Could not determine bundle name from url: {}", url);
}
return null;
}
}
10 changes: 5 additions & 5 deletions .../openhab/core/addon/marketplace/internal/community/CommunityRuleTemplateAddonHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,13 @@ public void install(Addon addon) throws MarketplaceHandlerException {
String yamlContent = (String) addon.getProperties().get(YAML_CONTENT_PROPERTY);

if (jsonDownloadUrl != null) {
marketplaceRuleTemplateProvider.addTemplateAsJSON(addon.getId(), getTemplateFromURL(jsonDownloadUrl));
marketplaceRuleTemplateProvider.addTemplateAsJSON(addon.getUid(), getTemplateFromURL(jsonDownloadUrl));
} else if (yamlDownloadUrl != null) {
marketplaceRuleTemplateProvider.addTemplateAsYAML(addon.getId(), getTemplateFromURL(yamlDownloadUrl));
marketplaceRuleTemplateProvider.addTemplateAsYAML(addon.getUid(), getTemplateFromURL(yamlDownloadUrl));
} else if (jsonContent != null) {
marketplaceRuleTemplateProvider.addTemplateAsJSON(addon.getId(), jsonContent);
marketplaceRuleTemplateProvider.addTemplateAsJSON(addon.getUid(), jsonContent);
} else if (yamlContent != null) {
marketplaceRuleTemplateProvider.addTemplateAsYAML(addon.getId(), yamlContent);
marketplaceRuleTemplateProvider.addTemplateAsYAML(addon.getUid(), yamlContent);
}
} catch (IOException e) {
logger.error("Rule template from marketplace cannot be downloaded: {}", e.getMessage());
Expand All @@ -96,7 +96,7 @@ public void install(Addon addon) throws MarketplaceHandlerException {

@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
marketplaceRuleTemplateProvider.getAll().stream().filter(t -> t.getTags().contains(addon.getId()))
marketplaceRuleTemplateProvider.getAll().stream().filter(t -> t.getTags().contains(addon.getUid()))
.forEach(w -> {
marketplaceRuleTemplateProvider.remove(w.getUID());
});
Expand Down
6 changes: 3 additions & 3 deletions .../org/openhab/core/addon/marketplace/internal/community/CommunityUIWidgetAddonHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ public void install(Addon addon) throws MarketplaceHandlerException {
String yamlContent = (String) addon.getProperties().get(YAML_CONTENT_PROPERTY);

if (yamlDownloadUrl != null) {
addWidgetAsYAML(addon.getId(), getWidgetFromURL(yamlDownloadUrl));
addWidgetAsYAML(addon.getUid(), getWidgetFromURL(yamlDownloadUrl));
} else if (yamlContent != null) {
addWidgetAsYAML(addon.getId(), yamlContent);
addWidgetAsYAML(addon.getUid(), yamlContent);
} else {
throw new IllegalArgumentException("Couldn't find the widget in the add-on entry");
}
Expand All @@ -96,7 +96,7 @@ public void install(Addon addon) throws MarketplaceHandlerException {

@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
widgetRegistry.getAll().stream().filter(w -> w.hasTag(addon.getId())).forEach(w -> {
widgetRegistry.getAll().stream().filter(w -> w.hasTag(addon.getUid())).forEach(w -> {
widgetRegistry.remove(w.getUID());
});
}
Expand Down
9 changes: 4 additions & 5 deletions ...lace/src/main/java/org/openhab/core/addon/marketplace/internal/json/JsonAddonService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,7 @@ protected List<Addon> getRemoteAddons() {

@Override
public @Nullable Addon getAddon(String id, @Nullable Locale locale) {
String fullId = ADDON_ID_PREFIX + id;
return cachedAddons.stream().filter(e -> fullId.equals(e.getId())).findAny().orElse(null);
return cachedAddons.stream().filter(e -> id.equals(e.getUid())).findAny().orElse(null);
}

@Override
Expand All @@ -147,9 +146,9 @@ protected List<Addon> getRemoteAddons() {
}

private Addon fromAddonEntry(AddonEntryDTO addonEntry) {
String fullId = ADDON_ID_PREFIX + addonEntry.id;
String uid = ADDON_ID_PREFIX + addonEntry.uid;
boolean installed = addonHandlers.stream().anyMatch(
handler -> handler.supports(addonEntry.type, addonEntry.contentType) && handler.isInstalled(fullId));
handler -> handler.supports(addonEntry.type, addonEntry.contentType) && handler.isInstalled(uid));

Map<String, Object> properties = new HashMap<>();
if (addonEntry.url.endsWith(".jar")) {
Expand All @@ -169,7 +168,7 @@ private Addon fromAddonEntry(AddonEntryDTO addonEntry) {
logger.debug("Failed to determine compatibility for addon {}: {}", addonEntry.id, e.getMessage());
}

return Addon.create(fullId).withType(addonEntry.type).withInstalled(installed)
return Addon.create(uid).withType(addonEntry.type).withId(addonEntry.id).withInstalled(installed)
.withDetailedDescription(addonEntry.description).withContentType(addonEntry.contentType)
.withAuthor(addonEntry.author).withVersion(addonEntry.version).withLabel(addonEntry.title)
.withCompatible(compatible).withMaturity(addonEntry.maturity).withProperties(properties)
Expand Down
1 change: 1 addition & 0 deletions ...e/src/main/java/org/openhab/core/addon/marketplace/internal/json/model/AddonEntryDTO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
* @author Jan N. Klug - Initial contribution
*/
public class AddonEntryDTO {
public String uid = "";
public String id = "";
public String type = "";
public String description = "";
Expand Down
4 changes: 2 additions & 2 deletions ...lace/src/test/java/org/openhab/core/addon/marketplace/AbstractRemoteAddonServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public void testAddonResultsAreCached() {
public void testAddonIsReportedAsInstalledIfStorageEntryMissing() {
addonService.setInstalled(TEST_ADDON);
List<Addon> addons = addonService.getAddons(null);
Addon addon = addons.stream().filter(a -> getFullAddonId(TEST_ADDON).equals(a.getId())).findAny().orElse(null);
Addon addon = addons.stream().filter(a -> getFullAddonId(TEST_ADDON).equals(a.getUid())).findAny().orElse(null);

assertThat(addon, notNullValue());
assertThat(addon.isInstalled(), is(true));
Expand All @@ -142,7 +142,7 @@ public void testInstalledAddonIsStillPresentAfterRemoteIsDisabledOrMissing() {
// check only the installed addon is present
addons = addonService.getAddons(null);
assertThat(addons, hasSize(1));
assertThat(addons.get(0).getId(), is(getFullAddonId(TEST_ADDON)));
assertThat(addons.get(0).getUid(), is(getFullAddonId(TEST_ADDON)));
}

@Test
Expand Down
Loading

1 comment on commit 6738277

@openhab-bot
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This commit has been mentioned on openHAB Community. There might be relevant details there:

https://community.openhab.org/t/binding-not-visible-after-starting-osgi/143614/4

Please sign in to comment.