IGNITE-16086 Support configurable tracked branches

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jobs/CheckQueueJob.java

@@ -40,10 +40,7 @@
 import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
 import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
-import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
-import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
-import org.apache.ignite.tcbot.engine.conf.ITrackedChain;
-import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
+import org.apache.ignite.tcbot.engine.conf.*;
 import org.apache.ignite.tcbot.notify.ISlackSender;
 import org.apache.ignite.tcbot.persistence.IStringCompactor;
 import org.apache.ignite.tcignited.ITeamcityIgnited;
@@ -92,6 +89,8 @@ public class CheckQueueJob implements Runnable {
     /** */
     @Inject private ITcBotConfig cfg;
 
+    @Inject private ITrackedBranchesConfig trackedBranchesConfig;
+
     /** */
     @Inject private ISlackSender slackSender;
 
@@ -146,7 +145,7 @@ protected String runEx() {
             return msg;
         }
 
-        Stream<ITrackedBranch> tracked = cfg.getTrackedBranches().branchesStream();
+        Stream<ITrackedBranch> tracked = trackedBranchesConfig.branchesStream();
 
         int srvsChecked = 0, chainsChecked = 0;
 

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotWebAppModule.java

@@ -29,17 +29,20 @@
 import org.apache.ignite.ci.observer.BuildObserver;
 import org.apache.ignite.ci.observer.ObserverTask;
 import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig;
+import org.apache.ignite.ci.tcbot.conf.MixedFilesAndDbTrackedBranchesConfig;
 import org.apache.ignite.ci.tcbot.issue.IssueDetector;
 import org.apache.ignite.ci.tcbot.trends.MasterTrendsService;
 import org.apache.ignite.ci.web.model.hist.VisasHistoryStorage;
 import org.apache.ignite.githubignited.GitHubIgnitedModule;
 import org.apache.ignite.jiraignited.JiraIgnitedModule;
 import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
+import org.apache.ignite.tcbot.common.conf.ITcServerConfigSupplier;
 import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
 import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
 import org.apache.ignite.tcbot.engine.TcBotEngineModule;
 import org.apache.ignite.tcbot.engine.cleaner.Cleaner;
 import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
 import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
 import org.apache.ignite.tcbot.notify.TcBotNotificationsModule;
 import org.apache.ignite.tcbot.persistence.TcBotPersistenceModule;
@@ -90,9 +93,13 @@ public class TcBotWebAppModule extends AbstractModule {
 
         // common services
         install(new TcBotEngineModule());
+
+        bind(LocalFilesBasedConfig.class).in(new SingletonScope());
         bind(ITcBotConfig.class).to(LocalFilesBasedConfig.class).in(new SingletonScope());
-        //todo remove duplication of instances for base and for overriden class
-        bind(IDataSourcesConfigSupplier.class).to(LocalFilesBasedConfig.class).in(new SingletonScope());
+        bind(IDataSourcesConfigSupplier.class).to(ITcBotConfig.class).in(new SingletonScope());
+        bind(ITcServerConfigSupplier.class).to(IDataSourcesConfigSupplier.class).in(new SingletonScope());
+        bind(ITrackedBranchesConfig.class).to(MixedFilesAndDbTrackedBranchesConfig.class).in(new SingletonScope());
+
         bind(MasterTrendsService.class).in(new SingletonScope());
         bind(ITcBotBgAuth.class).to(TcBotBgAuthImpl.class).in(new SingletonScope());
     }

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java

@@ -21,6 +21,7 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Objects;
 import java.util.Properties;
 
@@ -46,6 +47,11 @@
  *
  */
 public class LocalFilesBasedConfig implements ITcBotConfig {
+
+    public LocalFilesBasedConfig() {
+        System.out.println("LocalFilesBasedConfig");
+    }
+
     private static TcBotJsonConfig reloadConfig() {
         final File workDir = TcBotWorkDir.resolveWorkDir();
         final File file = new File(workDir, "branches.json");
@@ -64,6 +70,11 @@ protected TcBotJsonConfig getConfig() {
         return reloadConfig();
     }
 
+    @Override
+    public Collection<String> getConfiguredServerIds() {
+        return getConfig().getConfiguredServerIds();
+    }
+
     /** {@inheritDoc} */
     @Override public ITcServerConfig getTeamcityConfig(String srvCode) {
         return getConfig().getTcConfig(srvCode)
@@ -126,11 +137,6 @@ protected TcBotJsonConfig getConfig() {
         return confidence == null || confidence < 0 || confidence > 1 ? ITcBotConfig.DEFAULT_CONFIDENCE : confidence;
     }
 
-    @Override
-    public ITrackedBranchesConfig getTrackedBranches() {
-        return getConfig();
-    }
-
     @GuavaCached(softValues = true, expireAfterWriteSecs = 3 * 60)
     protected Properties loadOldAuthProps(String srvCode) {
         File workDir = TcBotWorkDir.resolveWorkDir();

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/MemoryTrackedBranches.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.tcbot.conf;
+
+import org.apache.ignite.tcbot.engine.conf.BranchTracked;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * Inmem only implementation for tests
+ */
+public class MemoryTrackedBranches implements ITrackedBranchesConfig {
+    /**
+     * Branches.
+     */
+    private List<BranchTracked> branches = new ArrayList<>();
+
+    @Override
+    public Stream<ITrackedBranch> branchesStream() {
+        return branches.stream().map(t -> t);
+    }
+
+
+    public void addBranch(BranchTracked branch) {
+        branches.add(branch);
+    }
+}

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/MixedFilesAndDbTrackedBranchesConfig.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.tcbot.conf;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
+import org.apache.ignite.tcbot.engine.conf.BranchTrackedPersisted;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
+import org.apache.ignite.tcbot.persistence.CacheConfigs;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class MixedFilesAndDbTrackedBranchesConfig implements ITrackedBranchesConfig {
+    public static final String TRACKED_BRANCHES = "trackedBranches";
+    @Inject
+    private LocalFilesBasedConfig filesBasedCfg;
+
+    /** Ignite provider. */
+    @Inject private Provider<Ignite> igniteProvider;
+
+
+    @Override
+    public Stream<ITrackedBranch> branchesStream() {
+        //todo internal cached version,
+       // @GuavaCached(softValues = true, expireAfterWriteSecs = 3 * 60)
+        IgniteCache<String, BranchTrackedPersisted> cache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(TRACKED_BRANCHES));
+
+        Map<String, BranchTrackedPersisted> res = new HashMap<>();
+        filesBasedCfg.getConfig()
+                .getBranches()
+                .stream()
+                .map(BranchTrackedPersisted::new).forEach(btp -> {
+            res.put(btp.name(), btp);
+        });
+
+        Map<String, BranchTrackedPersisted> dbValues = new HashMap<>();
+        cache.forEach((entry) -> {
+            String key = entry.getKey();
+            dbValues.put(key, entry.getValue());
+        });
+
+        res.putAll(dbValues); // override config all values by values from DB, enforcing soft del as a priority
+
+        return res.values().stream().filter(BranchTrackedPersisted::isAlive).map(v -> v);
+    }
+}

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java

@@ -37,6 +37,7 @@
 import org.apache.ignite.ci.issue.Issue;
 import org.apache.ignite.ci.issue.IssueKey;
 import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
 import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
 import org.apache.ignite.tcbot.engine.issue.IssueType;
 import org.apache.ignite.ci.jobs.CheckQueueJob;
@@ -107,6 +108,8 @@ public class IssueDetector {
     /** Config. */
     @Inject private ITcBotConfig cfg;
 
+    @Inject private ITrackedBranchesConfig trackedBranchesConfig;
+
     /** Email sender. */
     @Inject private IEmailSender emailSender;
 
@@ -198,8 +201,7 @@ protected String sendNewNotificationsEx() {
             })
             .peek(issue -> filteredBuildTs.incrementAndGet())
             .filter(issue -> {
-                return cfg.getTrackedBranches()
-                    .get(issue.trackedBranchName)
+                return trackedBranchesConfig.get(issue.trackedBranchName)
                     .filter(tb -> !tb.disableIssueTypes().contains(issue.type()))
                     .isPresent();
             })
@@ -594,7 +596,7 @@ public void startBackgroundCheck(ITcBotUserCreds prov) {
      *
      */
     private void checkFailures() {
-        cfg.getTrackedBranches().branchesStream().forEach(tb -> {
+        trackedBranchesConfig.branchesStream().forEach(tb -> {
             try {
                 checkFailuresEx(tb.name());
             } catch (Exception e) {

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java

@@ -66,6 +66,7 @@
 import org.apache.ignite.tcbot.common.conf.IJiraServerConfig;
 import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
 import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
 import org.apache.ignite.tcbot.engine.pr.BranchTicketMatcher;
 import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor;
 import org.apache.ignite.tcbot.engine.ui.ShortSuiteUi;
@@ -134,6 +135,8 @@ public class TcBotTriggerAndSignOffService {
     /** Config. */
     @Inject ITcBotConfig cfg;
 
+    @Inject ITrackedBranchesConfig trackedBranchesConfig;
+
     @Inject
     BranchTicketMatcher ticketMatcher;
 
@@ -686,8 +689,7 @@ private String findDefaultBuildType(String srvIdOrAlias) {
 
         String realTcId = Strings.isNullOrEmpty(tcCfg.reference()) ? srvIdOrAlias : tcCfg.reference();
 
-        cfg.getTrackedBranches()
-            .get(trBranch)
+        trackedBranchesConfig.get(trBranch)
             .ifPresent(
                 b -> b.chainsStream()
                     .filter(c -> Objects.equals(realTcId, c.serverCode()))

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetTrackedBranches.java

@@ -20,6 +20,7 @@
 import com.google.common.base.Strings;
 import com.google.inject.Injector;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -34,9 +35,10 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
+import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.common.conf.ITcServerConfigSupplier;
 import org.apache.ignite.tcbot.engine.conf.ChainAtServer;
 import org.apache.ignite.ci.tcbot.TcBotGeneralService;
-import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
 import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
 import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
 import org.apache.ignite.tcbot.engine.conf.ITrackedChain;
@@ -80,10 +82,10 @@ public List<String> getIdsIfAccessible() {
     @NotNull public Stream<ITrackedBranch> accessibleTrackedBranches() {
         ITcBotUserCreds prov = ITcBotUserCreds.get(req);
         Injector injector = CtxListener.getInjector(ctx);
-        ITcBotConfig cfg = injector.getInstance(ITcBotConfig.class);
+        ITrackedBranchesConfig trackedBranchesConfig = injector.getInstance(ITrackedBranchesConfig.class);
         ITeamcityIgnitedProvider tcProv = injector.getInstance(ITeamcityIgnitedProvider.class);
 
-        return cfg.getTrackedBranches().branchesStream()
+        return trackedBranchesConfig.branchesStream()
             .filter(bt ->
                 bt.chainsStream().anyMatch(chain -> tcProv.hasAccess(chain.serverCode(), prov)));
     }
@@ -109,10 +111,10 @@ public Set<ChainAtServer> getSuitesUnique(ITrackedBranchesConfig trackedBranches
     public Set<ChainAtServer> getSuites(@Nullable @QueryParam("server") String srvId) {
         ITcBotUserCreds prov = ITcBotUserCreds.get(req);
         Injector injector = CtxListener.getInjector(ctx);
-        ITcBotConfig cfg = injector.getInstance(ITcBotConfig.class);
+        ITrackedBranchesConfig trackedBranchesConfig = injector.getInstance(ITrackedBranchesConfig.class);
         ITeamcityIgnitedProvider tcProv = injector.getInstance(ITeamcityIgnitedProvider.class);
 
-        return getSuitesUnique(cfg.getTrackedBranches())
+        return getSuitesUnique(trackedBranchesConfig)
             .stream()
             .filter(chainAtSrv ->
                 Strings.isNullOrEmpty(srvId)
@@ -129,18 +131,18 @@ public Set<ChainAtServer> getSuites(@Nullable @QueryParam("server") String srvId
     public Set<String> getServerIds() {
         ITcBotUserCreds prov = ITcBotUserCreds.get(req);
         Injector injector = CtxListener.getInjector(ctx);
-        ITcBotConfig cfg = injector.getInstance(ITcBotConfig.class);
+        ITcServerConfigSupplier tcServerConfigSupplier = injector.getInstance(ITcServerConfigSupplier.class);
         ITeamcityIgnitedProvider tcProv = injector.getInstance(ITeamcityIgnitedProvider.class);
 
-        return cfg.getServerIds()
+        return tcServerConfigSupplier.getConfiguredServerIds()
             .stream()
             .filter(srvId -> tcProv.hasAccess(srvId, prov))
             .collect(Collectors.toSet());
     }
 
 
     /**
-     * Finds all registere unique teamcity branches.
+     * Finds all registered unique teamcity branches.
      * @param srvCodeOrAlias Server code or its alisas.
      */
     @GET
@@ -160,4 +162,19 @@ public Set<String> tcBranches(@Nullable @QueryParam("srvCode") String srvCodeOrA
             })
             .map(ITrackedChain::tcBranch).collect(Collectors.toSet());
     }
+
+    /**
+     * Finds all registered unique teamcity branches.
+     * @param code TC Bot's branch code
+     */
+    @GET
+    @Path("get")
+    public ITrackedBranch get(@Nullable @QueryParam("code") String code) {
+        String branchCode = Strings.isNullOrEmpty(code) ? ITcServerConfig.DEFAULT_TRACKED_BRANCH_NAME : code;
+
+        return accessibleTrackedBranches()
+                .filter(tb -> branchCode.equals(tb.name()))
+                .findAny()
+                .orElse(null);
+    }
 }