From b457b03798b067ebbd1fb3093b6d149b0b7f3b8d Mon Sep 17 00:00:00 2001 From: antkr Date: Mon, 3 Jun 2019 14:59:25 +0300 Subject: [PATCH 001/149] GG-14719 jackson dependency upgraded to 2.9.9 (cherry picked from commit b5f3f72) --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index 137fe4645f991..b81b251ca7d1a 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -74,7 +74,7 @@ 1.2 4.5.1 4.4.3 - 2.9.6 + 2.9.9 1.9.13 3.20.0-GA 1.0.0_1 From 53899a00bc8685f996f194573c061d7c2ab9c90a Mon Sep 17 00:00:00 2001 From: antkr Date: Mon, 3 Jun 2019 14:57:33 +0300 Subject: [PATCH 002/149] GG-14930 Upgraded Jetty dependency to 9.4.18 (cherry picked from commit 7883609) --- .../http/jetty/GridJettyRestProtocol.java | 20 ++++++++++--------- parent/pom.xml | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestProtocol.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestProtocol.java index d136eb590c558..c35457650425e 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestProtocol.java +++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestProtocol.java @@ -19,6 +19,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.net.BindException; import java.net.InetAddress; import java.net.SocketException; import java.net.URL; @@ -32,6 +33,7 @@ import org.apache.ignite.internal.processors.rest.protocols.GridRestProtocolAdapter; import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.IgniteSpiException; @@ -230,14 +232,6 @@ private boolean startJetty() throws IgniteCheckedException { return false; } - catch (SocketException ignore) { - if (log.isDebugEnabled()) - log.debug("Failed to bind HTTP server to configured port."); - - stopJetty(); - - return false; - } catch (MultiException e) { if (log.isDebugEnabled()) log.debug("Caught multi exception: " + e); @@ -254,7 +248,15 @@ private boolean startJetty() throws IgniteCheckedException { return false; } catch (Exception e) { - throw new IgniteCheckedException("Failed to start Jetty HTTP server.", e); + if (X.hasCause(e, BindException.class)) { + if (log.isDebugEnabled()) + log.debug("Failed to bind HTTP server to configured port."); + + stopJetty(); + + return false; + } else + throw new IgniteCheckedException("Failed to start Jetty HTTP server.", e); } } diff --git a/parent/pom.xml b/parent/pom.xml index b81b251ca7d1a..e31cd470c76e7 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -81,7 +81,7 @@ 1.0.1 1.0.0 16.0.3 - 9.4.11.v20180605 + 9.4.18.v20190429 1.13 1.1.1 0.1.53_1 From 422e6ceacedd51c0344ce330f1ab6f6c71a7774b Mon Sep 17 00:00:00 2001 From: antkr Date: Mon, 3 Jun 2019 14:55:43 +0300 Subject: [PATCH 003/149] GG-19121 commons-codec dependency upgraded to 1.12 (cherry picked from commit 7043f6c) --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index e31cd470c76e7..ba763b3232229 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -56,7 +56,7 @@ 2.22.0 1.9.2_1 1.9.3 - 1.11 + 1.12 3.2.2 2.6 2.2.5 From e33a69a1a4c2839850dd8d8e71126f5fa8359bcd Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Tue, 2 Jul 2019 11:53:31 +0300 Subject: [PATCH 004/149] GG-20558 Recovery mode for page memory used only for binary restore --- .../persistence/GridCacheDatabaseSharedManager.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index a566ae90b53eb..48c7cbf4f0465 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -2379,7 +2379,7 @@ private WALPointer performBinaryMemoryRestore( stripedApplyPage((pageMem) -> { try { - applyPageDelta(pageMem, pageDelta); + applyPageDelta(pageMem, pageDelta, true); applied.incrementAndGet(); } @@ -2575,24 +2575,25 @@ public void applyPageSnapshot(PageMemoryEx pageMem, PageSnapshot pageSnapshotRec /** * @param pageMem Page memory. * @param pageDeltaRecord Page delta record. + * @param restore Get page for restore. * @throws IgniteCheckedException If failed. */ - private void applyPageDelta(PageMemoryEx pageMem, PageDeltaRecord pageDeltaRecord) throws IgniteCheckedException { + private void applyPageDelta(PageMemoryEx pageMem, PageDeltaRecord pageDeltaRecord, boolean restore) throws IgniteCheckedException { int grpId = pageDeltaRecord.groupId(); long pageId = pageDeltaRecord.pageId(); // Here we do not require tag check because we may be applying memory changes after // several repetitive restarts and the same pages may have changed several times. - long page = pageMem.acquirePage(grpId, pageId, IoStatisticsHolderNoOp.INSTANCE, true); + long page = pageMem.acquirePage(grpId, pageId, IoStatisticsHolderNoOp.INSTANCE, restore); try { - long pageAddr = pageMem.writeLock(grpId, pageId, page, true); + long pageAddr = pageMem.writeLock(grpId, pageId, page, restore); try { pageDeltaRecord.applyDelta(pageMem, pageAddr); } finally { - pageMem.writeUnlock(grpId, pageId, page, null, true, true); + pageMem.writeUnlock(grpId, pageId, page, null, true, restore); } } finally { @@ -2840,7 +2841,7 @@ private RestoreLogicalState applyLogicalUpdates( stripedApplyPage((pageMem) -> { try { - applyPageDelta(pageMem, pageDelta); + applyPageDelta(pageMem, pageDelta, false); } catch (IgniteCheckedException e) { U.error(log, "Failed to apply page delta, " + pageDelta); From 859c40990c5453c1d469b836a1d83cb943358190 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Tue, 2 Jul 2019 12:45:24 +0300 Subject: [PATCH 005/149] GG-20536 Holding exception at the record reading until next call of iterator --- .../wal/AbstractWalRecordsIterator.java | 15 ++++++++++++++- .../IgniteAbstractWalIteratorInvalidCrcTest.java | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 8a38f28e607ed..298a5b59e09a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -60,6 +60,11 @@ public abstract class AbstractWalRecordsIterator */ protected IgniteBiTuple curRec; + /** + * The exception which can be thrown during reading next record. It holds until the next calling of next record. + */ + private IgniteCheckedException curException; + /** * Current WAL segment absolute index.
Determined as lowest number of file at start, is changed during advance * segment @@ -118,9 +123,17 @@ protected AbstractWalRecordsIterator( /** {@inheritDoc} */ @Override protected IgniteBiTuple onNext() throws IgniteCheckedException { + if (curException != null) + throw curException; + IgniteBiTuple ret = curRec; - advance(); + try { + advance(); + } + catch (IgniteCheckedException e) { + curException = e; + } return ret; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java index 98d89308930c0..7915565165a64 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java @@ -229,7 +229,7 @@ protected void doTest( // WAL iterator advances to the next record and only then returns current one, // so next record has to be valid as well. - assertEquals(lastReadPtr.next(), beforeCorruptedPtr); + assertEquals(lastReadPtr, beforeCorruptedPtr); } else try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) { From c41982f25d2f35401528bb12c955c92e484460c6 Mon Sep 17 00:00:00 2001 From: Andrey Kalinin Date: Fri, 14 Jun 2019 13:06:04 +0300 Subject: [PATCH 006/149] GG-14970 IdleVerify command should print end time of execution. (cherry picked from commit 1a0fd3e8c1933dfc067eb875a8f75e0f025257e8) --- .../ignite/internal/commandline/CommandHandler.java | 12 +++++++++++- .../apache/ignite/util/GridCommandHandlerTest.java | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 152fee0b16478..7ae90d31864bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.commandline; import java.io.File; +import java.time.Duration; import java.time.LocalDateTime; import java.util.Arrays; import java.util.Collections; @@ -205,12 +206,14 @@ public CommandHandler(Logger logger) { * @return Exit code. */ public int execute(List rawArgs) { + LocalDateTime startTime = LocalDateTime.now(); + Thread.currentThread().setName("session=" + ses); logger.info("Control utility [ver. " + ACK_VER_STR + "]"); logger.info(COPYRIGHT); logger.info("User: " + System.getProperty("user.name")); - logger.info("Time: " + LocalDateTime.now()); + logger.info("Time: " + startTime); String commandName = ""; @@ -312,6 +315,13 @@ public int execute(List rawArgs) { return EXIT_CODE_UNEXPECTED_ERROR; } finally { + LocalDateTime endTime = LocalDateTime.now(); + + Duration diff = Duration.between(startTime, endTime); + + logger.info("Control utility has completed execution at: " + endTime); + logger.info("Execution time: " + diff.toMillis() + " ms"); + Arrays.stream(logger.getHandlers()) .filter(handler -> handler instanceof FileHandler) .forEach(Handler::close); diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index d5801bcd3399b..089943f377c17 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -1123,6 +1123,18 @@ public void testHelp() { assertNotContains(log, testOutStr, "Control.sh"); } + /** */ + @Test + public void testPrintTimestampAtEndsOfExecution() { + injectTestSystemOut(); + + assertEquals(EXIT_CODE_OK, execute()); + + String testOutStr = testOut.toString(); + + assertContains(log, testOutStr, "Control utility has completed execution at: "); + } + /** * @throws Exception If failed. */ From 7719fe9c6cd586ab22403dd76b5933d163eefc25 Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Wed, 3 Jul 2019 11:31:58 +0300 Subject: [PATCH 007/149] GG-20761 [GG-20749] remove migration pendingTree compatibility test for 2.1 pds --- .../PdsWithTtlCompatibilityTest.java | 195 ------------------ .../IgniteCompatibilityBasicTestSuite.java | 3 - 2 files changed, 198 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/PdsWithTtlCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/PdsWithTtlCompatibilityTest.java index 946caddb5f203..e69de29bb2d1d 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/PdsWithTtlCompatibilityTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/PdsWithTtlCompatibilityTest.java @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.compatibility; - -import java.util.Collection; -import java.util.concurrent.TimeUnit; -import javax.cache.Cache; -import javax.cache.expiry.AccessedExpiryPolicy; -import javax.cache.expiry.Duration; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.cache.CacheWriteSynchronizationMode; -import org.apache.ignite.compatibility.persistence.IgnitePersistenceCompatibilityAbstractTest; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.MemoryConfiguration; -import org.apache.ignite.configuration.PersistentStoreConfiguration; -import org.apache.ignite.configuration.WALMode; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.persistence.migration.UpgradePendingTreeToPerPartitionTask; -import org.apache.ignite.internal.util.typedef.PA; -import org.apache.ignite.lang.IgniteFuture; -import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.testframework.GridTestUtils; - -/** - * Test PendingTree upgrading to per-partition basis. Test fill cache with persistence enabled and with ExpirePolicy - * configured on ignite-2.1 version and check if entries will be correctly expired when a new version node started. - * - * Note: Test for ignite-2.3 version will always fails due to entry ttl update fails with assertion on checkpoint lock - * check. - */ -public class PdsWithTtlCompatibilityTest extends IgnitePersistenceCompatibilityAbstractTest { - /** */ - static final String TEST_CACHE_NAME = PdsWithTtlCompatibilityTest.class.getSimpleName(); - - /** */ - static final int DURATION_SEC = 10; - - /** */ - private static final int ENTRIES_CNT = 100; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setPeerClassLoadingEnabled(false); - - cfg.setDataStorageConfiguration( - new DataStorageConfiguration() - .setDefaultDataRegionConfiguration( - new DataRegionConfiguration() - .setMaxSize(32L * 1024 * 1024) - .setPersistenceEnabled(true) - .setCheckpointPageBufferSize(16L * 1024 * 1024) - ).setWalMode(WALMode.LOG_ONLY)); - - return cfg; - } - - /** - * Tests opportunity to read data from previous Ignite DB version. - * - * @throws Exception If failed. - */ - public void testNodeStartByOldVersionPersistenceData_2_1() throws Exception { - doTestStartupWithOldVersion("2.1.0"); - } - - /** - * Tests opportunity to read data from previous Ignite DB version. - * - * @param igniteVer 3-digits version of ignite - * @throws Exception If failed. - */ - protected void doTestStartupWithOldVersion(String igniteVer) throws Exception { - try { - startGrid(1, igniteVer, new ConfigurationClosure(), new PostStartupClosure()); - - stopAllGrids(); - - IgniteEx ignite = startGrid(0); - - assertEquals(1, ignite.context().discovery().topologyVersion()); - - ignite.active(true); - - validateResultingCacheData(ignite, ignite.cache(TEST_CACHE_NAME)); - } - finally { - stopAllGrids(); - } - } - - /** - * @param cache to be filled by different keys and values. Results may be validated in {@link - * #validateResultingCacheData(Ignite, IgniteCache)}. - */ - public static void saveCacheData(Cache cache) { - for (int i = 0; i < ENTRIES_CNT; i++) - cache.put(i, "data-" + i); - - //Touch - for (int i = 0; i < ENTRIES_CNT; i++) - assertNotNull(cache.get(i)); - } - - /** - * Asserts cache contained all expected values as it was saved before. - * - * @param cache cache should be filled using {@link #saveCacheData(Cache)}. - */ - public static void validateResultingCacheData(Ignite ignite, - IgniteCache cache) throws IgniteInterruptedCheckedException { - - final long expireTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(DURATION_SEC + 1); - - final IgniteFuture> future = ignite.compute().broadcastAsync(new UpgradePendingTreeToPerPartitionTask()); - - GridTestUtils.waitForCondition(new PA() { - @Override public boolean apply() { - return future.isDone() && expireTime < System.currentTimeMillis(); - } - }, TimeUnit.SECONDS.toMillis(DURATION_SEC + 2)); - - for (Boolean res : future.get()) - assertTrue(res); - - for (int i = 0; i < ENTRIES_CNT; i++) - assertNull(cache.get(i)); - } - - /** */ - public static class ConfigurationClosure implements IgniteInClosure { - /** {@inheritDoc} */ - @Override public void apply(IgniteConfiguration cfg) { - cfg.setLocalHost("127.0.0.1"); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER); - - cfg.setDiscoverySpi(disco); - - cfg.setPeerClassLoadingEnabled(false); - - cfg.setMemoryConfiguration(new MemoryConfiguration().setDefaultMemoryPolicySize(256L * 1024 * 1024)); - cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setWalMode(WALMode.LOG_ONLY) - .setCheckpointingPageBufferSize(16L * 1024 * 1024)); - } - } - - /** */ - public static class PostStartupClosure implements IgniteInClosure { - /** {@inheritDoc} */ - @Override public void apply(Ignite ignite) { - ignite.active(true); - - CacheConfiguration cacheCfg = new CacheConfiguration<>(); - cacheCfg.setName(TEST_CACHE_NAME); - cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - cacheCfg.setBackups(1); - cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - cacheCfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, DURATION_SEC))); - cacheCfg.setEagerTtl(true); - cacheCfg.setGroupName("myGroup"); - - IgniteCache cache = ignite.createCache(cacheCfg); - - saveCacheData(cache); - - ignite.active(false); - } - } -} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java index fcfd5a7939ca4..eaa38afdd6d61 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java @@ -19,7 +19,6 @@ import junit.framework.TestSuite; import org.apache.ignite.compatibility.persistence.DummyPersistenceCompatibilityTest; -import org.apache.ignite.compatibility.PdsWithTtlCompatibilityTest; import org.apache.ignite.compatibility.persistence.FoldersReuseCompatibilityTest; import org.apache.ignite.compatibility.persistence.IgniteUuidCompatibilityTest; import org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest; @@ -37,8 +36,6 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(DummyPersistenceCompatibilityTest.class); - suite.addTestSuite(PdsWithTtlCompatibilityTest.class); - suite.addTestSuite(FoldersReuseCompatibilityTest.class); suite.addTestSuite(MigratingToWalV2SerializerWithCompactionTest.class); From 5a616de590a309f2e6b209b8df176d2f25c960e9 Mon Sep 17 00:00:00 2001 From: ktkalenko Date: Wed, 3 Jul 2019 12:16:36 +0300 Subject: [PATCH 008/149] GG-20420 NullPointerException If transaction failed and failure handler doesn't configured explicitly. (cherry picked from commit c8bd480f42a3f890bf2448bec735537fbfb3574c) --- .../processors/cache/transactions/IgniteTxAdapter.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index 418311edb6ed1..11772f96df2ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -754,18 +754,16 @@ public final IgniteCheckedException heuristicException(Throwable ex) { public void logTxFinishErrorSafe(@Nullable IgniteLogger log, boolean commit, Throwable e) { assert e != null : "Exception is expected"; - final String fmt = "Failed completing the transaction: [commit=%s, tx=%s, plc=%s]"; + final String fmt = "Failed completing the transaction: [commit=%s, tx=%s]"; try { // First try printing a full transaction. This is error prone. - U.error(log, String.format(fmt, commit, this, - cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e); + U.error(log, String.format(fmt, commit, this), e); } catch (Throwable e0) { e.addSuppressed(e0); - U.error(log, String.format(fmt, commit, CU.txString(this), - cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e); + U.error(log, String.format(fmt, commit, CU.txString(this)), e); } } From 3b97ac7b57acc9901c4747f811d19c765371512b Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Wed, 3 Jul 2019 17:04:45 +0300 Subject: [PATCH 009/149] GG-20193 fix IGNITE-9858 adapted into the branch (cherry-picked from commit #fa8e5da60546bf2e46cc16b13baf0a21c394e7b6) --- .../ignite/spi/discovery/tcp/ClientImpl.java | 2 +- .../ipfinder/TcpDiscoveryIpFinderAdapter.java | 14 +- .../TcpDiscoveryMulticastIpFinder.java | 213 +++++++++--------- .../TcpClientDiscoverySpiMulticastTest.java | 8 +- 4 files changed, 125 insertions(+), 112 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 2141819977868..58a09223c1130 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -292,7 +292,7 @@ class ClientImpl extends TcpDiscoveryImpl { sockReader = new SocketReader(); sockReader.start(); - if (spi.ipFinder.isShared()) + if (spi.ipFinder.isShared() && spi.isForceServerMode()) registerLocalNodeAddress(); msgWorker = new MessageWorker(log); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAdapter.java index 1cd91f63b9fae..09951048b9c7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAdapter.java @@ -40,7 +40,13 @@ public abstract class TcpDiscoveryIpFinderAdapter implements TcpDiscoveryIpFinde @GridToStringExclude private volatile IgniteSpiContext spiCtx; - /** Ignite instance . */ + /** + * Ignite instance. + * + * @deprecated Since 2.8. May contain an invalid Ignite instance when multiple nodes shares same + * {@link TcpDiscoveryIpFinder} instance. + */ + @Deprecated @IgniteInstanceResource @GridToStringExclude protected Ignite ignite; @@ -57,8 +63,7 @@ public abstract class TcpDiscoveryIpFinderAdapter implements TcpDiscoveryIpFinde /** {@inheritDoc} */ @Override public void initializeLocalAddresses(Collection addrs) throws IgniteSpiException { - if (!discoveryClientMode()) - registerAddresses(addrs); + registerAddresses(addrs); } /** {@inheritDoc} */ @@ -92,7 +97,10 @@ public TcpDiscoveryIpFinderAdapter setShared(boolean shared) { /** * @return {@code True} if TCP discovery works in client mode. + * @deprecated Since 2.8. May return incorrect value if client and server nodes shares same {@link + * TcpDiscoveryIpFinder} instance. */ + @Deprecated protected boolean discoveryClientMode() { boolean clientMode; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java index c59fa78fffbd9..f73f18a849982 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/multicast/TcpDiscoveryMulticastIpFinder.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; @@ -126,7 +127,7 @@ public class TcpDiscoveryMulticastIpFinder extends TcpDiscoveryVmIpFinder { @GridToStringExclude private InetAddress mcastAddr; - /** */ + /** Interfaces used to send requests. */ @GridToStringExclude private Set reqItfs; @@ -308,132 +309,66 @@ public int getTimeToLive() { /** {@inheritDoc} */ @Override public void initializeLocalAddresses(Collection addrs) throws IgniteSpiException { - // If IGNITE_OVERRIDE_MCAST_GRP system property is set, use its value to override multicast group from - // configuration. Used for testing purposes. - String overrideMcastGrp = System.getProperty(IGNITE_OVERRIDE_MCAST_GRP); - - if (overrideMcastGrp != null) - mcastGrp = overrideMcastGrp; - - if (F.isEmpty(mcastGrp)) - throw new IgniteSpiException("Multicast IP address is not specified."); - - if (mcastPort < 0 || mcastPort > 65535) - throw new IgniteSpiException("Invalid multicast port: " + mcastPort); - - if (resWaitTime <= 0) - throw new IgniteSpiException("Invalid wait time, value greater than zero is expected: " + resWaitTime); - - if (addrReqAttempts <= 0) - throw new IgniteSpiException("Invalid number of address request attempts, " + - "value greater than zero is expected: " + addrReqAttempts); - - if (ttl != -1 && (ttl < 0 || ttl > 255)) - throw new IgniteSpiException("Time-to-live value is out of 0 <= TTL <= 255 range: " + ttl); - - if (F.isEmpty(getRegisteredAddresses())) + if (F.isEmpty(super.getRegisteredAddresses())) U.warn(log, "TcpDiscoveryMulticastIpFinder has no pre-configured addresses " + "(it is recommended in production to specify at least one address in " + "TcpDiscoveryMulticastIpFinder.getAddresses() configuration property)"); - boolean clientMode = discoveryClientMode(); + Collection locAddrs = resolveLocalAddresses(); - try { - mcastAddr = InetAddress.getByName(mcastGrp); - } - catch (UnknownHostException e) { - throw new IgniteSpiException("Unknown multicast group: " + mcastGrp, e); - } + addrSnds = new ArrayList<>(locAddrs.size()); - if (!mcastAddr.isMulticastAddress()) - throw new IgniteSpiException("Invalid multicast group address: " + mcastAddr); + reqItfs = new HashSet<>(U.capacity(locAddrs.size())); // Interfaces used to send requests. - Collection locAddrs; + for (InetAddress addr : locAddrs) { + try { + addrSnds.add(new AddressSender(mcastAddr, addr, addrs)); - try { - locAddrs = U.resolveLocalAddresses(U.resolveLocalHost(locAddr)).get1(); - } - catch (IOException | IgniteCheckedException e) { - throw new IgniteSpiException("Failed to resolve local addresses [locAddr=" + locAddr + ']', e); + reqItfs.add(addr); + } + catch (IOException e) { + if (log.isDebugEnabled()) + log.debug("Failed to create multicast socket [mcastAddr=" + mcastAddr + + ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", locAddr=" + addr + + ", err=" + e + ']'); + } } - assert locAddrs != null; - - addrSnds = new ArrayList<>(locAddrs.size()); - - reqItfs = new HashSet<>(locAddrs.size()); // Interfaces used to send requests. - - for (String locAddr : locAddrs) { - InetAddress addr; + locNodeAddrs = new HashSet<>(addrs); + if (addrSnds.isEmpty()) { try { - addr = InetAddress.getByName(locAddr); + // Create non-bound socket if local host is loopback or failed to create sockets explicitly + // bound to interfaces. + addrSnds.add(new AddressSender(mcastAddr, null, addrs)); } - catch (UnknownHostException e) { + catch (IOException e) { if (log.isDebugEnabled()) - log.debug("Failed to resolve local address [locAddr=" + locAddr + ", err=" + e + ']'); - - continue; + log.debug("Failed to create multicast socket [mcastAddr=" + mcastAddr + + ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", err=" + e + ']'); } - if (!addr.isLoopbackAddress()) { + if (addrSnds.isEmpty()) { try { - if (!clientMode) - addrSnds.add(new AddressSender(mcastAddr, addr, addrs)); + addrSnds.add(new AddressSender(mcastAddr, mcastAddr, addrs)); - reqItfs.add(addr); + reqItfs.add(mcastAddr); } catch (IOException e) { if (log.isDebugEnabled()) log.debug("Failed to create multicast socket [mcastAddr=" + mcastAddr + - ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", locAddr=" + addr + + ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", locAddr=" + mcastAddr + ", err=" + e + ']'); } } } - if (!clientMode) { - locNodeAddrs = new HashSet<>(addrs); - - if (addrSnds.isEmpty()) { - try { - // Create non-bound socket if local host is loopback or failed to create sockets explicitly - // bound to interfaces. - addrSnds.add(new AddressSender(mcastAddr, null, addrs)); - } - catch (IOException e) { - if (log.isDebugEnabled()) - log.debug("Failed to create multicast socket [mcastAddr=" + mcastAddr + - ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", err=" + e + ']'); - } - - if (addrSnds.isEmpty()) { - try { - addrSnds.add(new AddressSender(mcastAddr, mcastAddr, addrs)); - - reqItfs.add(mcastAddr); - } - catch (IOException e) { - if (log.isDebugEnabled()) - log.debug("Failed to create multicast socket [mcastAddr=" + mcastAddr + - ", mcastGrp=" + mcastGrp + ", mcastPort=" + mcastPort + ", locAddr=" + mcastAddr + - ", err=" + e + ']'); - } - } - } - - if (!addrSnds.isEmpty()) { - for (AddressSender addrSnd : addrSnds) - addrSnd.start(); - } - else - mcastErr = true; - } - else { - assert addrSnds.isEmpty() : addrSnds; - - locNodeAddrs = Collections.emptySet(); + if (!addrSnds.isEmpty()) { + for (AddressSender addrSnd : addrSnds) + addrSnd.start(); } + else + mcastErr = true; } /** {@inheritDoc} */ @@ -445,6 +380,9 @@ public int getTimeToLive() { /** {@inheritDoc} */ @Override public synchronized Collection getRegisteredAddresses() { + if (mcastAddr == null) + reqItfs = new HashSet<>(resolveLocalAddresses()); + if (mcastAddr != null && reqItfs != null) { Collection ret; @@ -460,7 +398,7 @@ public int getTimeToLive() { if (ret.isEmpty()) { if (mcastErr && firstReq) { - if (getRegisteredAddresses().isEmpty()) { + if (super.getRegisteredAddresses().isEmpty()) { InetSocketAddress addr = new InetSocketAddress("localhost", TcpDiscoverySpi.DFLT_PORT); U.quietAndWarn(log, "TcpDiscoveryMulticastIpFinder failed to initialize multicast, " + @@ -482,6 +420,77 @@ public int getTimeToLive() { return super.getRegisteredAddresses(); } + /** + * Resolve local addresses. + * + * @return List of non-loopback addresses. + */ + private Collection resolveLocalAddresses() { + // If IGNITE_OVERRIDE_MCAST_GRP system property is set, use its value to override multicast group from + // configuration. Used for testing purposes. + String overrideMcastGrp = System.getProperty(IGNITE_OVERRIDE_MCAST_GRP); + + if (overrideMcastGrp != null) + mcastGrp = overrideMcastGrp; + + if (F.isEmpty(mcastGrp)) + throw new IgniteSpiException("Multicast IP address is not specified."); + + if (mcastPort < 0 || mcastPort > 65535) + throw new IgniteSpiException("Invalid multicast port: " + mcastPort); + + if (resWaitTime <= 0) + throw new IgniteSpiException("Invalid wait time, value greater than zero is expected: " + resWaitTime); + + if (addrReqAttempts <= 0) + throw new IgniteSpiException("Invalid number of address request attempts, " + + "value greater than zero is expected: " + addrReqAttempts); + + if (ttl != -1 && (ttl < 0 || ttl > 255)) + throw new IgniteSpiException("Time-to-live value is out of 0 <= TTL <= 255 range: " + ttl); + + try { + mcastAddr = InetAddress.getByName(mcastGrp); + } + catch (UnknownHostException e) { + throw new IgniteSpiException("Unknown multicast group: " + mcastGrp, e); + } + + if (!mcastAddr.isMulticastAddress()) + throw new IgniteSpiException("Invalid multicast group address: " + mcastAddr); + + Collection locAddrs; + + try { + locAddrs = U.resolveLocalAddresses(U.resolveLocalHost(locAddr)).get1(); + } + catch (IOException | IgniteCheckedException e) { + throw new IgniteSpiException("Failed to resolve local addresses [locAddr=" + locAddr + ']', e); + } + + assert locAddrs != null; + + List inetAddrs = new ArrayList<>(locAddrs.size()); + + for (String locAddr : locAddrs) { + InetAddress addr; + + try { + addr = InetAddress.getByName(locAddr); + } + catch (UnknownHostException e) { + if (log.isDebugEnabled()) + log.debug("Failed to resolve local address [locAddr=" + locAddr + ", err=" + e + ']'); + + continue; + } + + if (!addr.isLoopbackAddress()) + inetAddrs.add(addr); + } + + return inetAddrs; + } /** * @param reqItfs Interfaces used to send requests. @@ -770,7 +779,7 @@ private class AddressReceiver extends IgniteSpiThread { * @param sockAddr Optional address multicast socket should be bound to. */ private AddressReceiver(InetAddress mcastAddr, InetAddress sockAddr) { - super(ignite == null ? null : ignite.name(), "tcp-disco-multicast-addr-rcvr", log); + super(null, "tcp-disco-multicast-addr-rcvr", log); this.mcastAddr = mcastAddr; this.sockAddr = sockAddr; } @@ -814,7 +823,7 @@ private class AddressSender extends IgniteSpiThread { */ private AddressSender(InetAddress mcastGrp, @Nullable InetAddress sockItf, Collection addrs) throws IOException { - super(ignite == null ? null : ignite.name(), "tcp-disco-multicast-addr-sender", log); + super(null, "tcp-disco-multicast-addr-sender", log); this.mcastGrp = mcastGrp; this.addrs = addrs; this.sockItf = sockItf; diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiMulticastTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiMulticastTest.java index e19b121aebdb7..17664444d5649 100644 --- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiMulticastTest.java +++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiMulticastTest.java @@ -24,6 +24,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.spi.discovery.DiscoverySpi; @@ -211,11 +212,6 @@ private void assertSpi(Ignite ignite, boolean client) { Collection addrSnds = GridTestUtils.getFieldValue(spi0.getIpFinder(), "addrSnds"); - assertNotNull(addrSnds); - - if (client) - assertTrue(addrSnds.isEmpty()); // Check client does not send its address. - else - assertFalse(addrSnds.isEmpty()); + assertEquals(client, F.isEmpty(addrSnds)); } } \ No newline at end of file From 58510ca8c307eb8ce12c6896d3b027384d4aaa7f Mon Sep 17 00:00:00 2001 From: Aleksei Scherbakov Date: Wed, 3 Jul 2019 17:34:36 +0300 Subject: [PATCH 010/149] GG-20702 Fix invalid partition clearing. (cherry picked from commit cafab0ded6fb5b6632826fe1b71d6fdde5a97823) # Conflicts: # modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java # modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java --- .../GridDhtPartitionsExchangeFuture.java | 14 +- .../dht/topology/GridDhtLocalPartition.java | 2 + .../GridDhtPartitionTopologyImpl.java | 86 +++++----- .../dht/topology/PartitionsEvictManager.java | 2 +- ...ePdsCacheWalDisabledOnRebalancingTest.java | 148 ++++++++++++++++-- .../junits/common/GridCommonAbstractTest.java | 5 +- 6 files changed, 203 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 5f3188fc3145a..1fb6702e036f1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -198,6 +198,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte /** */ private AtomicBoolean added = new AtomicBoolean(false); + /** Exchange type. */ + private volatile ExchangeType exchangeType; + /** * Discovery event receive latch. There is a race between discovery event processing and single message * processing, so it is possible to create an exchange future before the actual discovery event is received. @@ -478,6 +481,13 @@ public void affinityChangeMessage(CacheAffinityChangeMessage affChangeMsg) { return isDone() ? result() : exchCtx.events().topologyVersion(); } + /** + * @return Exchange type or null if not determined yet. + */ + public ExchangeType exchangeType() { + return exchangeType; + } + /** * Retreives the node which has WAL history since {@code cntrSince}. * @@ -843,6 +853,8 @@ else if (msg instanceof WalStateAbstractMessage) cctx.cache().registrateProxyRestart(resolveCacheRequests(exchActions), afterLsnrCompleteFut); + exchangeType = exchange; + for (PartitionsExchangeAware comp : cctx.exchange().exchangeAwareComponents()) comp.onInitBeforeTopologyLock(this); @@ -5067,7 +5079,7 @@ public void cleanUp() { /** * */ - enum ExchangeType { + public enum ExchangeType { /** */ CLIENT, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 354a7851a61b1..39d457bb94257 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -588,6 +588,8 @@ private boolean casState(long state, GridDhtPartitionState toState) { */ public boolean own() { while (true) { + assert !clear : "Could not own clearing partition " + this; + long state = this.state.get(); GridDhtPartitionState partState = getPartState(state); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index d51f053597857..e9588a8de0c54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -75,6 +75,7 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; +import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.ExchangeType.ALL; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; @@ -764,58 +765,61 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) { long updateSeq = this.updateSeq.incrementAndGet(); - for (int p = 0; p < partitions; p++) { - GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true); + // Skip partition updates in case of not real exchange. + if (!ctx.localNode().isClient() && exchFut.exchangeType() == ALL) { + for (int p = 0; p < partitions; p++) { + GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true); - if (partitionLocalNode(p, topVer)) { - // Prepare partition to rebalance if it's not happened on full map update phase. - if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED) - locPart = rebalancePartition(p, true, exchFut); + if (partitionLocalNode(p, topVer)) { + // Prepare partition to rebalance if it's not happened on full map update phase. + if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED) + locPart = rebalancePartition(p, true, exchFut); - GridDhtPartitionState state = locPart.state(); + GridDhtPartitionState state = locPart.state(); - if (state == MOVING) { - if (grp.rebalanceEnabled()) { - Collection owners = owners(p); + if (state == MOVING) { + if (grp.rebalanceEnabled()) { + Collection owners = owners(p); + + // If an owner node left during exchange, then new exchange should be started with detecting lost partitions. + if (!F.isEmpty(owners)) { + if (log.isDebugEnabled()) + log.debug("Will not own partition (there are owners to rebalance from) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", owners = " + owners + ']'); + } - // If an owner node left during exchange, then new exchange should be started with detecting lost partitions. - if (!F.isEmpty(owners)) { - if (log.isDebugEnabled()) - log.debug("Will not own partition (there are owners to rebalance from) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", owners = " + owners + ']'); + // It's important to clear non empty moving partitions before full rebalancing. + // Consider the scenario: + // Node1 has keys k1 and k2 in the same partition. + // Node2 started rebalancing from Node1. + // Node2 received k1, k2 and failed before moving partition to OWNING state. + // Node1 removes k2 but update has not been delivered to Node1 because of failure. + // After new full rebalance Node1 will only send k1 to Node2 causing lost removal. + // NOTE: avoid calling clearAsync for partition twice per topology version. + // TODO FIXME clearing is not always needed see IGNITE-11799 + if (grp.persistenceEnabled() && !exchFut.isHistoryPartition(grp, locPart.id()) && + !locPart.isClearing() && !locPart.isEmpty()) + locPart.clearAsync(); } - - // It's important to clear non empty moving partitions before full rebalancing. - // Consider the scenario: - // Node1 has keys k1 and k2 in the same partition. - // Node2 started rebalancing from Node1. - // Node2 received k1, k2 and failed before moving partition to OWNING state. - // Node1 removes k2 but update has not been delivered to Node1 because of failure. - // After new full rebalance Node1 will only send k1 to Node2 causing lost removal. - // NOTE: avoid calling clearAsync for partition twice per topology version. - // TODO FIXME clearing is not always needed see IGNITE-11799 - if (grp.persistenceEnabled() && !exchFut.isHistoryPartition(grp, locPart.id()) && - !locPart.isClearing() && !locPart.isEmpty()) - locPart.clearAsync(); + else + updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); } - else - updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); } - } - else { - if (locPart != null) { - GridDhtPartitionState state = locPart.state(); + else { + if (locPart != null) { + GridDhtPartitionState state = locPart.state(); - if (state == MOVING) { - locPart.rent(false); + if (state == MOVING) { + locPart.rent(false); - updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); + updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); - changed = true; + changed = true; - if (log.isDebugEnabled()) { - log.debug("Evicting " + state + " partition (it does not belong to affinity) [" + - "grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']'); + if (log.isDebugEnabled()) { + log.debug("Evicting " + state + " partition (it does not belong to affinity) [" + + "grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']'); + } } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 826902cee1a87..704c7db3887f7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -369,7 +369,7 @@ private void showProgress() { ", grpId=" + grp.groupId() + ", remainingPartsToEvict=" + (totalTasks.get() - taskInProgress) + ", partsEvictInProgress=" + taskInProgress + - ", totalParts= " + grp.topology().localPartitions().size() + "]"); + ", totalParts=" + grp.topology().localPartitions().size() + "]"); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java index 1246db1090cae..6be608c78279a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java @@ -17,14 +17,19 @@ package org.apache.ignite.internal.processors.cache.persistence.db; import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.OpenOption; import java.nio.file.Paths; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.function.BiFunction; import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; @@ -38,6 +43,10 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.mxbean.CacheGroupMetricsMXBean; @@ -70,6 +79,15 @@ public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstra /** */ private static final String CACHE3_NAME = "cache3"; + /** Function to generate cache values. */ + private static final BiFunction GENERATING_FUNC = (s, i) -> s + "_value_" + i; + + /** */ + private static final Semaphore fileIoBlockingSemaphore = new Semaphore(Integer.MAX_VALUE); + + /** */ + private boolean useBlockingFileIO; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -122,6 +140,9 @@ public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstra .setPersistenceEnabled(true) .setMaxSize(256 * 1024 * 1024)); + if (useBlockingFileIO) + dsCfg.setFileIOFactory(new BlockingCheckpointFileIOFactory()); + cfg.setDataStorageConfiguration(dsCfg); } @@ -143,8 +164,8 @@ public void testClientJoinsLeavesDuringRebalancing() throws Exception { ig0.active(true); - for (int i = 0; i < 3; i++) - fillCache(ig0.getOrCreateCache("cache" + i), CACHE_SIZE); + for (int i = 1; i < 4; i++) + fillCache(ig0.dataStreamer("cache" + i), CACHE_SIZE, GENERATING_FUNC); String ig1Name = "node01-" + grid(1).localNode().consistentId(); @@ -188,7 +209,7 @@ public void testClientJoinsLeavesDuringRebalancing() throws Exception { /** * If server nodes from BLT leave topology and then join again after additional keys were put to caches, * rebalance starts. - * + * * Test verifies that all moving partitions get owned after rebalance finishes. * * @throws Exception If failed. @@ -196,7 +217,7 @@ public void testClientJoinsLeavesDuringRebalancing() throws Exception { public void testServerNodesFromBltLeavesAndJoinsDuringRebalancing() throws Exception { Ignite ig0 = startGridsMultiThreaded(4); - fillCache(ig0.cache(CACHE3_NAME), CACHE_SIZE); + fillCache(ig0.dataStreamer(CACHE3_NAME), CACHE_SIZE, GENERATING_FUNC); List nonAffinityKeys1 = nearKeys(grid(1).cache(CACHE3_NAME), 100, CACHE_SIZE / 2); List nonAffinityKeys2 = nearKeys(grid(2).cache(CACHE3_NAME), 100, CACHE_SIZE / 2); @@ -209,7 +230,7 @@ public void testServerNodesFromBltLeavesAndJoinsDuringRebalancing() throws Excep nonAffinityKeysSet.addAll(nonAffinityKeys1); nonAffinityKeysSet.addAll(nonAffinityKeys2); - fillCache(ig0.cache(CACHE3_NAME), nonAffinityKeysSet); + fillCache(ig0.dataStreamer(CACHE3_NAME), nonAffinityKeysSet, GENERATING_FUNC); int groupId = ((IgniteEx) ig0).cachex(CACHE3_NAME).context().groupId(); @@ -238,6 +259,97 @@ public void testServerNodesFromBltLeavesAndJoinsDuringRebalancing() throws Excep " partitions in MOVING state", allOwned); } + /** + * Scenario: when rebalanced MOVING partitions are owning by checkpointer, + * concurrent no-op exchange should not trigger partition clearing. + * + * @throws Exception If failed. + */ + public void testRebalancedPartitionsOwningWithAffinitySwitch() throws Exception { + Ignite ig0 = startGridsMultiThreaded(4); + fillCache(ig0.dataStreamer(CACHE3_NAME), CACHE_SIZE, GENERATING_FUNC); + + // Stop idx=2 to prepare for baseline topology change later. + stopGrid(2); + + // Stop idx=1 and cleanup LFS to trigger full rebalancing after it restart. + String ig1Name = "node01-" + grid(1).localNode().consistentId(); + stopGrid(1); + cleanPersistenceFiles(ig1Name); + + // Blocking fileIO and blockMessagePredicate to block checkpointer and rebalancing for node idx=1. + useBlockingFileIO = true; + + // Enable blocking checkpointer on node idx=1 (see BlockingCheckpointFileIOFactory). + fileIoBlockingSemaphore.drainPermits(); + + // Wait for rebalance (all partitions will be in MOVING state until cp is finished). + startGrid(1).cachex(CACHE3_NAME).context().group().preloader().rebalanceFuture().get(); + + startGrid("client"); + + fileIoBlockingSemaphore.release(Integer.MAX_VALUE); + + awaitPartitionMapExchange(false, false, null, true); + + assertPartitionsSame(idleVerify(grid(0), CACHE3_NAME)); + } + + /** FileIOFactory implementation that enables blocking of writes to disk so checkpoint can be blocked. */ + private static class BlockingCheckpointFileIOFactory implements FileIOFactory { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Delegate factory. */ + private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + FileIO delegate = delegateFactory.create(file, modes); + + return new FileIODecorator(delegate) { + @Override public int write(ByteBuffer srcBuf) throws IOException { + if (Thread.currentThread().getName().contains("checkpoint")) { + try { + fileIoBlockingSemaphore.acquire(); + } + catch (InterruptedException ignored) { + // No-op. + } + } + + return delegate.write(srcBuf); + } + + @Override public int write(ByteBuffer srcBuf, long position) throws IOException { + if (Thread.currentThread().getName().contains("checkpoint")) { + try { + fileIoBlockingSemaphore.acquire(); + } + catch (InterruptedException ignored) { + // No-op. + } + } + + return delegate.write(srcBuf, position); + } + + @Override public int write(byte[] buf, int off, int len) throws IOException { + if (Thread.currentThread().getName().contains("checkpoint")) { + try { + fileIoBlockingSemaphore.acquire(); + } + catch (InterruptedException ignored) { + // No-op. + } + } + + return delegate.write(buf, off, len); + } + }; + } + } + /** */ private void cleanPersistenceFiles(String igName) throws Exception { String ig1DbPath = Paths.get(DFLT_STORE_DIR, igName).toString(); @@ -253,14 +365,30 @@ private void cleanPersistenceFiles(String igName) throws Exception { } /** */ - private void fillCache(IgniteCache cache, int cacheSize) { + private void fillCache( + IgniteDataStreamer streamer, + int cacheSize, + BiFunction generatingFunc + ) { + String name = streamer.cacheName(); + for (int i = 0; i < cacheSize; i++) - cache.put(i, "value_" + i); + streamer.addData(i, generatingFunc.apply(name, i)); + + streamer.close(); } /** */ - private void fillCache(IgniteCache cache, Collection keys) { + private void fillCache( + IgniteDataStreamer streamer, + Collection keys, + BiFunction generatingFunc + ) { + String cacheName = streamer.cacheName(); + for (Integer key : keys) - cache.put(key, "value_" + key); + streamer.addData(key, generatingFunc.apply(cacheName, key)); + + streamer.close(); } } diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 3641b5e3ba8df..fd1b3b564f863 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -859,6 +859,8 @@ protected void printPartitionState(String cacheName, int firstParts) { sb.append("nodeId=") .append(k.context().localNodeId()) + .append(" consistentId=") + .append(k.localNode().consistentId()) .append(" isDone=") .append(syncFut.isDone()) .append("\n"); @@ -939,7 +941,8 @@ protected void printPartitionState(String cacheName, int firstParts) { .append(part == null ? "NA" : part.dataStore().partUpdateCounter()) .append(" fullSize=") .append(part == null ? "NA" : part.fullSize()) - .append(" state=").append(part.state()); + .append(" state=").append(part.state()) + .append(" reservations=").append(part.reservations()); } else sb.append(p).append(" is null"); From 1c9dbe22ca9a446c0a4fac657d6a87bad2878fad Mon Sep 17 00:00:00 2001 From: korlov Date: Wed, 3 Jul 2019 14:30:10 +0300 Subject: [PATCH 011/149] GG-20464 backport to 8.5-master of GG-19518 IoStatisticsBasicIndexSelfTest --- .../processors/query/h2/database/H2Tree.java | 5 +++++ .../index/IoStatisticsBasicIndexSelfTest.java | 15 +++++++++++---- .../IgniteCacheWithIndexingTestSuite.java | 3 +++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java index 1790134dd6270..29c1022ac1df0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java @@ -323,6 +323,11 @@ public int compareRows(SearchRow r1, SearchRow r2) { return 0; } + /** {@inheritDoc} */ + @Override protected IoStatisticsHolder statisticsHolder() { + return stats; + } + /** * @param v1 First value. * @param v2 Second value. diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IoStatisticsBasicIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IoStatisticsBasicIndexSelfTest.java index c7b0fa5cbfcd6..5faf579e9b57a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IoStatisticsBasicIndexSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/IoStatisticsBasicIndexSelfTest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; @@ -77,11 +78,17 @@ public class IoStatisticsBasicIndexSelfTest extends GridCommonAbstractTest { fields.put("valLong", Long.class.getName()); fields.put("valPojo", Pojo.class.getName()); + Set keyFields = new HashSet<>(); + keyFields.add("keyStr"); + keyFields.add("keyLong"); + keyFields.add("keyPojo"); + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME) .setQueryEntities(Collections.singleton( new QueryEntity() .setKeyType(Key.class.getName()) .setValueType(Val.class.getName()) + .setKeyFields(keyFields) .setFields(fields) .setIndexes(indexes) )); @@ -170,14 +177,14 @@ private void checkStat() { Assert.assertEquals(PK_HASH_INDEXES, hashIndexes); - Set sortedIndexCaches = ioStat.deriveStatisticNames(IoStatisticsType.SORTED_INDEX); + Set sortedIdxCaches = ioStat.deriveStatisticNames(IoStatisticsType.SORTED_INDEX); - Assert.assertEquals(1, sortedIndexCaches.size()); + Assert.assertEquals(1, sortedIdxCaches.size()); Set sortedIdxNames = ioStat.deriveStatisticSubNames(IoStatisticsType.SORTED_INDEX, - sortedIndexCaches.toArray()[0].toString()); + sortedIdxCaches.toArray()[0].toString()); - Assert.assertEquals(sortedIndexCaches.toString(), indexes.size() + NUMBER_OF_PK_SORTED_INDEXES, sortedIdxNames.size()); + Assert.assertEquals(sortedIdxCaches.toString(), indexes.size() + NUMBER_OF_PK_SORTED_INDEXES, sortedIdxNames.size()); for (String idxName : sortedIdxNames) { Long logicalReads = ioStat.logicalReads(IoStatisticsType.SORTED_INDEX, DEFAULT_CACHE_NAME, idxName); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index bd1c99fb0d24d..05e91e1c8751d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheGroupsSqlTest; import org.apache.ignite.internal.processors.cache.IgniteCacheStarvationOnRebalanceTest; import org.apache.ignite.internal.processors.cache.IgniteClientReconnectQueriesTest; +import org.apache.ignite.internal.processors.cache.index.IoStatisticsBasicIndexSelfTest; import org.apache.ignite.internal.processors.cache.persistence.RebuildIndexLogMessageTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicLocalSelfTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicPartitionedSelfTest; @@ -95,6 +96,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(RebuildIndexLogMessageTest.class); + suite.addTestSuite(IoStatisticsBasicIndexSelfTest.class); + return suite; } } From b823cc6757b6c09b11e0658a6daf398eb2030941 Mon Sep 17 00:00:00 2001 From: Aleksei Scherbakov Date: Wed, 3 Jul 2019 19:54:18 +0300 Subject: [PATCH 012/149] GG-20798 Fix failing tests. --- .../dht/preloader/GridDhtPartitionDemander.java | 2 +- .../dht/topology/GridDhtPartitionTopologyImpl.java | 5 ++--- .../transactions/TxPartitionCounterStatePutTest.java | 12 ++++-------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java index d0a63211489ea..5179f075bb74e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java @@ -1082,7 +1082,7 @@ public static class RebalanceFuture extends GridFutureAdapter { this.log = null; this.rebalanceId = -1; this.routines = 0; - this.cancelLock = null; + this.cancelLock = new ReentrantReadWriteLock(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index e9588a8de0c54..19ae255344f44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -1591,9 +1591,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD if (exchangeVer != null && nodeMap != null && grp.persistenceEnabled() && - readyTopVer.initialized()) { - - assert exchFut != null; + readyTopVer.initialized() && + exchFut != null) { for (Map.Entry e : nodeMap.entrySet()) { int p = e.getKey(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStatePutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStatePutTest.java index 69019d0137dca..209f4eb24b309 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStatePutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPartitionCounterStatePutTest.java @@ -133,16 +133,14 @@ public void testPutTxConcurrentPersistent() throws Exception { doTestPutConcurrent(TX_CACHE, true); } - /** */ + /** TODO https://issues.apache.org/jira/browse/IGNITE-11793 */ public void testPutAtomicConcurrentPersistentWithIsolatedMode() throws Exception { // doTestPutConcurrent(ATOMIC_CACHE, false); - fail("https://issues.apache.org/jira/browse/IGNITE-11793"); } - /** */ + /** TODO https://issues.apache.org/jira/browse/IGNITE-11793 */ public void testPutTxConcurrentPersistentWithIsolatedMode() throws Exception { //doTestPutConcurrent(TX_CACHE, false); - fail("https://issues.apache.org/jira/browse/IGNITE-11793"); } /** */ @@ -165,16 +163,14 @@ public void testPutTxConcurrentVolatile() throws Exception { doTestPutConcurrent(TX_CACHE_MEMORY, true); } - /** */ + /** TODO https://issues.apache.org/jira/browse/IGNITE-11793 */ public void testPutAtomicConcurrentVolatileWithIsolatedMode() throws Exception { // doTestPutConcurrent(ATOMIC_CACHE_MEMORY, false); - fail("https://issues.apache.org/jira/browse/IGNITE-11793"); } - /** */ + /** TODO https://issues.apache.org/jira/browse/IGNITE-11793 */ public void testPutTxConcurrentVolatileWithIsolatedMode() throws Exception { // doTestPutConcurrent(TX_CACHE_MEMORY, false); - fail("https://issues.apache.org/jira/browse/IGNITE-11793"); } /** */ From 44091fa2103acf1c0c72f5bbd53bcbff690f948c Mon Sep 17 00:00:00 2001 From: Andrey Kalinin Date: Mon, 1 Jul 2019 15:34:51 +0300 Subject: [PATCH 013/149] GG-19492 Add debug output for o.a.i.internal.processors.cache.persistence.GridCacheOffheapManager#restorePartitionStates method. (cherry picked from commit 621f6afb0983bcb52eaf5f71ade74cd453dd1749) --- .../persistence/GridCacheOffheapManager.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 5d6b53ae813cf..2e73094761469 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -511,6 +511,8 @@ else if (needSnapshot) for (int p = 0; p < grp.affinity().partitions(); p++) { Integer recoverState = partitionRecoveryStates.get(new GroupPartitionId(grp.groupId(), p)); + long startTime = U.currentTimeMillis(); + if (ctx.pageStore().exists(grp.groupId(), p)) { ctx.pageStore().ensure(grp.groupId(), p); @@ -555,7 +557,8 @@ else if (needSnapshot) if (log.isDebugEnabled()) log.debug("Restored partition state (from WAL) " + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + "]"); + ", updCntr=" + part.initialUpdateCounter() + + ", size=" + part.fullSize() + "]"); } else { int stateId = (int) io.getPartitionState(pageAddr); @@ -565,7 +568,8 @@ else if (needSnapshot) if (log.isDebugEnabled()) log.debug("Restored partition state (from page memory) " + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + "]"); + ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + + ", size=" + part.fullSize() + "]"); } } finally { @@ -590,13 +594,19 @@ else if (recoverState != null) { // Pre-create partition if having valid state. if (log.isDebugEnabled()) log.debug("Restored partition state (from WAL) " + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + "]"); + ", updCntr=" + part.initialUpdateCounter() + + ", size=" + part.fullSize() + "]"); } else { if (log.isDebugEnabled()) log.debug("Skipping partition on recovery (no page store OR wal state) " + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + "]"); } + + if (log.isDebugEnabled()) + log.debug("Finished restoring partition state " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + + ", time=" + (U.currentTimeMillis() - startTime) + " ms]"); } partitionStatesRestored = true; From b01ce115266b0f688a708023eb2053443b5232e7 Mon Sep 17 00:00:00 2001 From: ktkalenko Date: Thu, 4 Jul 2019 12:13:02 +0300 Subject: [PATCH 014/149] GG-18891 CommandHandlerParsingTest#testExperimentalCommandIsDisabled() doesn't work. (cherry picked from commit 93507600635af077d3cc6aaa1e297c92b0302da0) --- .../internal/commandline/WalCommands.java | 13 ++++++++- .../CommandHandlerParsingTest.java | 28 ++++++++----------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java index e2489cf0c99a1..f892e985b5065 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java @@ -69,8 +69,9 @@ public class WalCommands implements Command> { */ private String walArgs; + /** {@inheritDoc} */ @Override public void printUsage(Logger logger) { - if (IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false)) { + if (enableExperimental()) { Command.usage(logger, "Print absolute paths of unused archived wal segments on each node:", WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]"); Command.usage(logger, "Delete unused archived wal segments on each node:", WAL, WAL_DELETE, @@ -115,6 +116,9 @@ public class WalCommands implements Command> { /** {@inheritDoc} */ @Override public void parseArguments(CommandArgIterator argIter) { + if (!enableExperimental()) + throw new IllegalArgumentException("Experimental command is disabled."); + String str = argIter.nextArg("Expected arguments for " + WAL.text()); String walAct = str.toLowerCase(); @@ -268,4 +272,11 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) { @Override public String name() { return WAL.toCommandName(); } + + /** + * @return Value of {@link IgniteSystemProperties#IGNITE_ENABLE_EXPERIMENTAL_COMMAND} + */ + private boolean enableExperimental() { + return IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 4235f00fda842..1d9df94b083f6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -276,23 +276,19 @@ private void generateAllCombinations(List res, List source, Predicate< public void testExperimentalCommandIsDisabled() { System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND); - try { - parseArgs(Arrays.asList(WAL.text(), WAL_PRINT)); - } - catch (Throwable e) { - e.printStackTrace(); - - assertTrue(e instanceof IllegalArgumentException); - } - - try { - parseArgs(Arrays.asList(WAL.text(), WAL_DELETE)); - } - catch (Throwable e) { - e.printStackTrace(); + GridTestUtils.assertThrows( + null, + () -> parseArgs(Arrays.asList(WAL.text(), WAL_PRINT)), + IllegalArgumentException.class, + null + ); - assertTrue(e instanceof IllegalArgumentException); - } + GridTestUtils.assertThrows( + null, + () -> parseArgs(Arrays.asList(WAL.text(), WAL_DELETE)), + IllegalArgumentException.class, + null + ); } /** From ad9070d89e99f5feadc09c60fb59f71e7574cae0 Mon Sep 17 00:00:00 2001 From: korlov Date: Thu, 4 Jul 2019 12:28:09 +0300 Subject: [PATCH 015/149] GG-20810 backport to 8.5-master of GG-20807 Fix failing IoStatisticsBasicIndexSelfTest --- .../IgniteCacheWithIndexingAndPersistenceTestSuite.java | 2 ++ .../ignite/testsuites/IgniteCacheWithIndexingTestSuite.java | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java index 2f3ddecef0d66..7eef9318c3771 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java @@ -19,6 +19,7 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.StartCachesInParallelTest; +import org.apache.ignite.internal.processors.cache.index.IoStatisticsBasicIndexSelfTest; import org.apache.ignite.util.GridCommandHandlerIndexingTest; /** @@ -34,6 +35,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(GridCommandHandlerIndexingTest.class); suite.addTestSuite(StartCachesInParallelTest.class); + suite.addTestSuite(IoStatisticsBasicIndexSelfTest.class); return suite; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index 05e91e1c8751d..bd1c99fb0d24d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheGroupsSqlTest; import org.apache.ignite.internal.processors.cache.IgniteCacheStarvationOnRebalanceTest; import org.apache.ignite.internal.processors.cache.IgniteClientReconnectQueriesTest; -import org.apache.ignite.internal.processors.cache.index.IoStatisticsBasicIndexSelfTest; import org.apache.ignite.internal.processors.cache.persistence.RebuildIndexLogMessageTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicLocalSelfTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicPartitionedSelfTest; @@ -96,8 +95,6 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(RebuildIndexLogMessageTest.class); - suite.addTestSuite(IoStatisticsBasicIndexSelfTest.class); - return suite; } } From 3616b6a7767a39206445af579073f313ef5a5e8b Mon Sep 17 00:00:00 2001 From: ibelyakov Date: Fri, 5 Jul 2019 15:03:40 +0300 Subject: [PATCH 016/149] GG-19515: GG: destroying of the cache and re-creating with the same name make DR paused in case of persistence. (cherry picked from commit e093716171736fca9c47a42df584338d4c29a511) --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 +- .../internal/processors/cache/dr/GridCacheDrManager.java | 3 +-- .../internal/processors/cache/dr/GridOsCacheDrManager.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 1fb6702e036f1..368e1c80ceda7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2173,7 +2173,7 @@ private String exchangeTimingsLogMessage(String header, List timings) { if (drCacheCtx.isDrEnabled()) { try { - drCacheCtx.dr().onExchange(res, exchId.isLeft(), activateCluster()); + drCacheCtx.dr().onExchange(res, exchId.isLeft()); } catch (IgniteCheckedException e) { U.error(log, "Failed to notify DR: " + e, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java index 33a52a11a03d1..f2a4b30c4af82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java @@ -60,10 +60,9 @@ public void replicate(KeyCacheObject key, * * @param topVer Topology version. * @param left {@code True} if exchange has been caused by node leave. - * @param activate {@code True} if exchange has been caused by cluster activation. * @throws IgniteCheckedException If failed. */ - public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException; + public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException; /** * @return {@code True} is DR is enabled. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java index 425e79c536344..f3c1b23f7c7d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java @@ -78,7 +78,7 @@ public class GridOsCacheDrManager implements GridCacheDrManager { } /** {@inheritDoc} */ - @Override public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException { + @Override public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException { // No-op. } From b04ec7e511056e24242e67cb5e0da7f9be39db42 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Fri, 5 Jul 2019 18:34:23 +0300 Subject: [PATCH 017/149] GG-20822: Compact log and drop log level down to info for cancelled queries. (cherry picked from commit 03283da8cbb7c70b0fed497d3c0e58ec010b6d52) --- .../h2/twostep/GridMapQueryExecutor.java | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 15ae7652c1978..290d146f45bb0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -583,23 +583,28 @@ private void onQueryRequest0( if (lazy) stopAndUnregisterCurrentLazyWorker(); - GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class); - - if (retryErr != null) { - final String retryCause = String.format( - "Failed to execute non-collocated query (will retry) [localNodeId=%s, rmtNodeId=%s, reqId=%s, " + - "errMsg=%s]", ctx.localNodeId(), node.id(), reqId, retryErr.getMessage() - ); - - sendRetry(node, reqId, segmentId, retryCause); + if (e instanceof QueryCancelledException) { + sendError(node, reqId, e); } else { - U.error(log, "Failed to execute local query.", e); + GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class); - sendError(node, reqId, e); + if (retryErr != null) { + final String retryCause = String.format( + "Failed to execute non-collocated query (will retry) [localNodeId=%s, rmtNodeId=%s, reqId=%s, " + + "errMsg=%s]", ctx.localNodeId(), node.id(), reqId, retryErr.getMessage() + ); - if (e instanceof Error) - throw (Error)e; + sendRetry(node, reqId, segmentId, retryCause); + } + else { + U.error(log, "Failed to execute local query.", e); + + sendError(node, reqId, e); + + if (e instanceof Error) + throw (Error)e; + } } } finally { @@ -733,7 +738,17 @@ private void sendError(ClusterNode node, long qryReqId, Throwable err) { GridQueryFailResponse msg = new GridQueryFailResponse(qryReqId, err); if (node.isLocal()) { - U.error(log, "Failed to run map query on local node.", err); + if (err instanceof QueryCancelledException) { + String errMsg = "Failed to run cancelled map query on local node: [localNodeId=" + + node.id() + ", reqId=" + qryReqId + ']'; + + if (log.isDebugEnabled()) + U.warn(log, errMsg, err); + else + log.info(errMsg); + } + else + U.error(log, "Failed to run map query on local node.", err); h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), msg); } From 21de580adeb899d052fe7cbe79eb7dda76140f40 Mon Sep 17 00:00:00 2001 From: ibelyakov Date: Sun, 7 Jul 2019 18:39:24 +0300 Subject: [PATCH 018/149] Revert "GG-19515: GG: destroying of the cache and re-creating with the same name make DR paused in case of persistence." This reverts commit 3616b6a7767a39206445af579073f313ef5a5e8b. --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 +- .../internal/processors/cache/dr/GridCacheDrManager.java | 3 ++- .../internal/processors/cache/dr/GridOsCacheDrManager.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 368e1c80ceda7..1fb6702e036f1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2173,7 +2173,7 @@ private String exchangeTimingsLogMessage(String header, List timings) { if (drCacheCtx.isDrEnabled()) { try { - drCacheCtx.dr().onExchange(res, exchId.isLeft()); + drCacheCtx.dr().onExchange(res, exchId.isLeft(), activateCluster()); } catch (IgniteCheckedException e) { U.error(log, "Failed to notify DR: " + e, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java index f2a4b30c4af82..33a52a11a03d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java @@ -60,9 +60,10 @@ public void replicate(KeyCacheObject key, * * @param topVer Topology version. * @param left {@code True} if exchange has been caused by node leave. + * @param activate {@code True} if exchange has been caused by cluster activation. * @throws IgniteCheckedException If failed. */ - public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException; + public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException; /** * @return {@code True} is DR is enabled. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java index f3c1b23f7c7d6..425e79c536344 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java @@ -78,7 +78,7 @@ public class GridOsCacheDrManager implements GridCacheDrManager { } /** {@inheritDoc} */ - @Override public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException { + @Override public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException { // No-op. } From b773b6c49c8ddab5c51a972afa48b86155f8da8c Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Mon, 8 Jul 2019 14:31:54 +0300 Subject: [PATCH 019/149] GG-21061 [GG-21057] Fixed IgniteCacheReplicatedQuerySelfTest.testNodeLeft flaky failed on TC (cherry picked from commit fe0ec74478d0c513f25b306e384319ac9220933a) --- .../IgniteCacheReplicatedQuerySelfTest.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java index 13942c2ede3d4..85748aa7d957a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java @@ -51,6 +51,7 @@ import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CachePeekMode.ALL; +import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; /** @@ -346,14 +347,13 @@ public void testLostIterator() throws Exception { /** * @throws Exception If failed. */ - @IgniteIgnore(value = "https://issues.apache.org/jira/browse/IGNITE-613", forceFailure = true) public void testNodeLeft() throws Exception { - Ignite g = startGrid("client"); + Ignite client = startGrid("client"); try { - assertTrue(g.configuration().isClientMode()); + assertTrue(client.configuration().isClientMode()); - IgniteCache cache = jcache(Integer.class, Integer.class); + IgniteCache cache = jcache(client, Integer.class, Integer.class); for (int i = 0; i < 1000; i++) cache.put(i, i); @@ -373,18 +373,21 @@ public void testNodeLeft() throws Exception { assertEquals(1, mapNode1.size() + mapNode2.size() + mapNode3.size()); - final UUID nodeId = g.cluster().localNode().id(); + final UUID nodeId = client.cluster().localNode().id(); - final CountDownLatch latch = new CountDownLatch(1); + final CountDownLatch latch = new CountDownLatch(3); - grid(0).events().localListen(new IgnitePredicate() { - @Override public boolean apply(Event evt) { - if (((DiscoveryEvent)evt).eventNode().id().equals(nodeId)) - latch.countDown(); + // Add listeners on all nodes. + for (int i = 0; i < 3; i++) { + grid(i).events().localListen(new IgnitePredicate() { + @Override public boolean apply(Event evt) { + if (((DiscoveryEvent)evt).eventNode().id().equals(nodeId)) + latch.countDown(); - return true; - } - }, EVT_NODE_LEFT); + return true; + } + }, EVT_NODE_LEFT, EVT_NODE_FAILED); + } stopGrid("client"); From d8a004f7077cb285cf3316633f1d90bf9536eb67 Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Mon, 8 Jul 2019 15:17:28 +0300 Subject: [PATCH 020/149] GG-21074 [GG-21055] Fixed IgniteClusterActivateDeactivateTest.testDeactivateSimple_5_Servers_5_Clients_FromClient flaky failed on TC (cherry picked from commit 25789310de2a00b96a443fe2b02fb1322356061e) --- .../IgniteClusterActivateDeactivateTest.java | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index e8b2d0a5ebb84..523904bc282c0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -320,7 +320,18 @@ private void rolloverSegmentAtLeastTwice(int activateFrom) { * @param nodes Number of nodes. * @param caches Number of caches. */ - final void checkCaches(int nodes, int caches) { + final void checkCaches(int nodes, int caches) throws InterruptedException { + checkCaches(nodes, caches, true); + } + + /** + * @param nodes Number of nodes. + * @param caches Number of caches. + */ + final void checkCaches(int nodes, int caches, boolean awaitExchange) throws InterruptedException { + if (awaitExchange) + awaitPartitionMapExchange(); + for (int i = 0; i < nodes; i++) { for (int c = 0; c < caches; c++) { IgniteCache cache = ignite(i).cache(CACHE_NAME_PREFIX + c); @@ -667,9 +678,6 @@ private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws startGrid(i); } - if (persistenceEnabled()) - ignite(deactivateFrom).cluster().active(true); - ignite(deactivateFrom).cluster().active(true); // Should be no-op. checkCaches(srvs + clients, CACHES); @@ -712,7 +720,7 @@ private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); } - checkCaches1(srvs + clients + 2); + checkCaches(srvs + clients + 2); } /** @@ -751,11 +759,11 @@ public void testClientReconnectClusterActive() throws Exception { checkCache(client, CU.UTILITY_CACHE_NAME, true); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, null); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); this.client = false; @@ -765,7 +773,7 @@ public void testClientReconnectClusterActive() throws Exception { startGrid(SRVS + CLIENTS + 1); - checkCaches1(SRVS + CLIENTS + 2); + checkCaches(SRVS + CLIENTS + 2); } /** @@ -794,7 +802,7 @@ public void testClientReconnectClusterInactive() throws Exception { checkCache(client, CU.UTILITY_CACHE_NAME, true); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); this.client = false; @@ -804,7 +812,7 @@ public void testClientReconnectClusterInactive() throws Exception { startGrid(SRVS + CLIENTS + 1); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); } /** @@ -842,7 +850,7 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws checkCache(client, CU.UTILITY_CACHE_NAME, true); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); // Wait for late affinity assignment to finish. awaitPartitionMapExchange(); @@ -892,7 +900,7 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws assertTrue(client.cluster().active()); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); checkCache(client, CACHE_NAME_PREFIX + 0, true); @@ -904,7 +912,7 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws startGrid(SRVS + CLIENTS + 1); - checkCaches1(SRVS + CLIENTS + 2); + checkCaches(SRVS + CLIENTS + 2); } /** @@ -979,7 +987,7 @@ private void clientReconnectClusterActivated(final boolean transition) throws Ex checkCache(client, CU.UTILITY_CACHE_NAME, true); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); checkCache(client, CACHE_NAME_PREFIX + 0, true); @@ -991,7 +999,7 @@ private void clientReconnectClusterActivated(final boolean transition) throws Ex startGrid(SRVS + CLIENTS + 1); - checkCaches1(SRVS + CLIENTS + 2); + checkCaches(SRVS + CLIENTS + 2); } /** @@ -1033,7 +1041,7 @@ public void testInactiveTopologyChanges() throws Exception { ignite(0).cluster().active(true); - checkCaches1(SRVS + CLIENTS); + checkCaches(SRVS + CLIENTS); checkRecordedMessages(true); @@ -1047,7 +1055,7 @@ public void testInactiveTopologyChanges() throws Exception { checkRecordedMessages(true); - checkCaches1(SRVS + CLIENTS + 2); + checkCaches(SRVS + CLIENTS + 2); } /** @@ -1104,7 +1112,7 @@ private void stateChangeFailover1(boolean activate) throws Exception { ignite(0).cluster().active(true); } - checkCaches1(9); + checkCaches(9); } /** @@ -1172,7 +1180,7 @@ private void stateChangeFailover2(boolean activate) throws Exception { ignite(0).cluster().active(true); } - checkCaches1(10); + checkCaches(10); } /** @@ -1239,7 +1247,7 @@ protected void doFinalChecks() throws Exception { for (int i = 0; i < 4; i++) startGrid(i); - checkCaches1(6); + checkCaches(6); } /** @@ -1306,8 +1314,8 @@ private void checkRecordedMessages(boolean exp) { /** * @param nodes Expected nodes number. */ - private void checkCaches1(int nodes) { - checkCaches(nodes, 2); + private void checkCaches(int nodes) throws InterruptedException { + checkCaches(nodes, 2, false); } /** From dfff292fc093253767730755a79b32d758264a41 Mon Sep 17 00:00:00 2001 From: ktkalenko Date: Mon, 8 Jul 2019 15:36:03 +0300 Subject: [PATCH 021/149] GG-20185 control.sh validate_indexes SQL Index issue must contain information about cache group. (cherry picked from commit 82409a97a786d367d880600580b2d63bc659a292) --- .../ignite/internal/visor/verify/ValidateIndexesClosure.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index f7a96bb4947f5..10591752058fd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -738,7 +738,7 @@ else if (current++ % checkThrough > 0) } } - String uniqueIdxName = "[cache=" + ctx.name() + ", idx=" + idx.getName() + "]"; + String uniqueIdxName = "[cacheGroup=" + ctx.group().name() + ", cache=" + ctx.name() + ", idx=" + idx.getName() + "]"; processedIndexes.incrementAndGet(); From e093eeb4d749be476901073b99f2da41b18d67eb Mon Sep 17 00:00:00 2001 From: tledkov Date: Mon, 8 Jul 2019 18:11:06 +0300 Subject: [PATCH 022/149] GG-20917 [8.5-next]-[GG-20908] Set max possible maintenance version for all x.x-master branches --- modules/core/src/main/resources/ignite.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties index d06256418df52..fa0a6771aa0d2 100644 --- a/modules/core/src/main/resources/ignite.properties +++ b/modules/core/src/main/resources/ignite.properties @@ -15,7 +15,7 @@ # limitations under the License. # -ignite.version=2.5.6-SNAPSHOT +ignite.version=2.5.127-SNAPSHOT ignite.build=0 ignite.revision=DEV ignite.rel.date=01011970 From 0adf555d42e6381bbe7b8ea2e5c99834aa2cc8b9 Mon Sep 17 00:00:00 2001 From: denis-chudov Date: Tue, 2 Jul 2019 18:05:48 +0300 Subject: [PATCH 023/149] GG-20729 Add ability to set default parallelism of rebuild indexes in ignite system properties (cherry picked from commit 05cdf467d97e3ee111c0bf89e9f51b12d5d82109) --- .../apache/ignite/IgniteSystemProperties.java | 8 ++++++++ .../schema/SchemaIndexCacheVisitorImpl.java | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 4f8d06238099f..7920a9cd6a572 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1121,6 +1121,14 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_LOG_CLASSPATH_CONTENT_ON_STARTUP = "IGNITE_LOG_CLASSPATH_CONTENT_ON_STARTUP"; + /** + * Index rebuilding parallelism level. If specified, sets the count of threads that are used for index rebuilding + * and can only be greater than 0, otherwise default value will be used. Maximum count of threads + * can't be greater than total available processors count. + * Default value is minimum of 4 and processors count / 4, but always greater than 0. + */ + public static final String INDEX_REBUILDING_PARALLELISM = "INDEX_REBUILDING_PARALLELISM"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java index 43e3ccf31a685..bf8cf417e5f76 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.schema; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -38,6 +39,7 @@ import java.util.List; +import static org.apache.ignite.IgniteSystemProperties.INDEX_REBUILDING_PARALLELISM; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; @@ -49,8 +51,7 @@ @SuppressWarnings("ForLoopReplaceableByForEach") public class SchemaIndexCacheVisitorImpl implements SchemaIndexCacheVisitor { /** Default degree of parallelism. */ - private static final int DFLT_PARALLELISM = - Math.min(4, Math.max(1, Runtime.getRuntime().availableProcessors() / 4)); + private static final int DFLT_PARALLELISM; /** Count of rows, being processed within a single checkpoint lock. */ private static final int BATCH_SIZE = 1000; @@ -70,6 +71,15 @@ public class SchemaIndexCacheVisitorImpl implements SchemaIndexCacheVisitor { /** Whether to stop the process. */ private volatile boolean stop; + static { + int parallelism = IgniteSystemProperties.getInteger(INDEX_REBUILDING_PARALLELISM, 0); + + if (parallelism > 0) + DFLT_PARALLELISM = Math.min(parallelism, Runtime.getRuntime().availableProcessors()); + else + DFLT_PARALLELISM = Math.min(4, Math.max(1, Runtime.getRuntime().availableProcessors() / 4)); + } + /** * Constructor. * @param cctx Cache context. @@ -94,7 +104,7 @@ public SchemaIndexCacheVisitorImpl(GridCacheContext cctx, SchemaIndexCacheFilter if (parallelism > 0) this.parallelism = Math.min(Runtime.getRuntime().availableProcessors(), parallelism); else - this.parallelism = DFLT_PARALLELISM; + this.parallelism = DFLT_PARALLELISM; if (cctx.isNear()) cctx = ((GridNearCacheAdapter)cctx.cache()).dht().context(); From ca61a704e3c58b0bba285930cbbef2de0b119980 Mon Sep 17 00:00:00 2001 From: ibessonov Date: Tue, 9 Jul 2019 17:27:44 +0300 Subject: [PATCH 024/149] GG-20845 Added explicit indexes to WALRecord.RecordType. --- .../pagemem/wal/record/WALRecord.java | 136 +++++++++++------- .../wal/serializer/RecordV1Serializer.java | 8 +- .../wal/serializer/RecordV2Serializer.java | 2 +- .../pagemem/wal/record/WALRecordTest.java | 41 ++++++ .../testsuites/IgniteBasicTestSuite.java | 3 + 5 files changed, 132 insertions(+), 58 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java index 0031b22efa478..60e64e97e97f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java @@ -32,126 +32,126 @@ public abstract class WALRecord { */ public enum RecordType { /** */ - TX_RECORD, + TX_RECORD (0), /** */ - PAGE_RECORD, + PAGE_RECORD (1), /** */ - DATA_RECORD, + DATA_RECORD (2), /** Checkpoint (begin) record */ - CHECKPOINT_RECORD, + CHECKPOINT_RECORD (3), /** WAL segment header record. */ - HEADER_RECORD, + HEADER_RECORD (4), // Delta records. /** */ - INIT_NEW_PAGE_RECORD, + INIT_NEW_PAGE_RECORD (5), /** */ - DATA_PAGE_INSERT_RECORD, + DATA_PAGE_INSERT_RECORD (6), /** */ - DATA_PAGE_INSERT_FRAGMENT_RECORD, + DATA_PAGE_INSERT_FRAGMENT_RECORD (7), /** */ - DATA_PAGE_REMOVE_RECORD, + DATA_PAGE_REMOVE_RECORD (8), /** */ - DATA_PAGE_SET_FREE_LIST_PAGE, + DATA_PAGE_SET_FREE_LIST_PAGE (9), /** */ - BTREE_META_PAGE_INIT_ROOT, + BTREE_META_PAGE_INIT_ROOT (10), /** */ - BTREE_META_PAGE_ADD_ROOT, + BTREE_META_PAGE_ADD_ROOT (11), /** */ - BTREE_META_PAGE_CUT_ROOT, + BTREE_META_PAGE_CUT_ROOT (12), /** */ - BTREE_INIT_NEW_ROOT, + BTREE_INIT_NEW_ROOT (13), /** */ - BTREE_PAGE_RECYCLE, + BTREE_PAGE_RECYCLE (14), /** */ - BTREE_PAGE_INSERT, + BTREE_PAGE_INSERT (15), /** */ - BTREE_FIX_LEFTMOST_CHILD, + BTREE_FIX_LEFTMOST_CHILD (16), /** */ - BTREE_FIX_COUNT, + BTREE_FIX_COUNT (17), /** */ - BTREE_PAGE_REPLACE, + BTREE_PAGE_REPLACE (18), /** */ - BTREE_PAGE_REMOVE, + BTREE_PAGE_REMOVE (19), /** */ - BTREE_PAGE_INNER_REPLACE, + BTREE_PAGE_INNER_REPLACE (20), /** */ - BTREE_FIX_REMOVE_ID, + BTREE_FIX_REMOVE_ID (21), /** */ - BTREE_FORWARD_PAGE_SPLIT, + BTREE_FORWARD_PAGE_SPLIT (22), /** */ - BTREE_EXISTING_PAGE_SPLIT, + BTREE_EXISTING_PAGE_SPLIT (23), /** */ - BTREE_PAGE_MERGE, + BTREE_PAGE_MERGE (24), /** */ - PAGES_LIST_SET_NEXT, + PAGES_LIST_SET_NEXT (25), /** */ - PAGES_LIST_SET_PREVIOUS, + PAGES_LIST_SET_PREVIOUS (26), /** */ - PAGES_LIST_INIT_NEW_PAGE, + PAGES_LIST_INIT_NEW_PAGE (27), /** */ - PAGES_LIST_ADD_PAGE, + PAGES_LIST_ADD_PAGE (28), /** */ - PAGES_LIST_REMOVE_PAGE, + PAGES_LIST_REMOVE_PAGE (29), /** */ - META_PAGE_INIT, + META_PAGE_INIT (30), /** */ - PARTITION_META_PAGE_UPDATE_COUNTERS, + PARTITION_META_PAGE_UPDATE_COUNTERS (31), /** Memory recovering start marker */ - MEMORY_RECOVERY, + MEMORY_RECOVERY (32), /** */ - TRACKING_PAGE_DELTA, + TRACKING_PAGE_DELTA (33), /** Meta page update last successful snapshot id. */ - META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID, + META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID (34), /** Meta page update last successful full snapshot id. */ - META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID, + META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID (35), /** Meta page update next snapshot id. */ - META_PAGE_UPDATE_NEXT_SNAPSHOT_ID, + META_PAGE_UPDATE_NEXT_SNAPSHOT_ID (36), /** Meta page update last allocated index. */ - META_PAGE_UPDATE_LAST_ALLOCATED_INDEX, + META_PAGE_UPDATE_LAST_ALLOCATED_INDEX (37), /** Partition meta update state. */ - PART_META_UPDATE_STATE, + PART_META_UPDATE_STATE (38), /** Page list meta reset count record. */ - PAGE_LIST_META_RESET_COUNT_RECORD, + PAGE_LIST_META_RESET_COUNT_RECORD (39), /** Switch segment record. * Marker record for indicate end of segment. @@ -160,41 +160,71 @@ public enum RecordType { * that one byte in the end,then we write SWITCH_SEGMENT_RECORD as marker end of segment. * No need write CRC or WAL pointer for this record. It is byte marker record. * */ - SWITCH_SEGMENT_RECORD, + SWITCH_SEGMENT_RECORD (40), /** */ - DATA_PAGE_UPDATE_RECORD, + DATA_PAGE_UPDATE_RECORD (41), /** init */ - BTREE_META_PAGE_INIT_ROOT2, + BTREE_META_PAGE_INIT_ROOT2 (42), /** Partition destroy. */ - PARTITION_DESTROY, + PARTITION_DESTROY (43), /** Snapshot record. */ - SNAPSHOT, + SNAPSHOT (44), /** Metastore data record. */ - METASTORE_DATA_RECORD, + METASTORE_DATA_RECORD (45), /** Exchange record. */ - EXCHANGE, + EXCHANGE (46), /** Reserved for future record. */ - RESERVED, + RESERVED (47), /** Rollback tx record. */ - ROLLBACK_TX_RECORD, + ROLLBACK_TX_RECORD (57), /** */ - PARTITION_META_PAGE_UPDATE_COUNTERS_V2; + PARTITION_META_PAGE_UPDATE_COUNTERS_V2 (58); + + /** Index for serialization. Should be consistent throughout all versions. */ + private final int idx; + + /** + * @param idx Index for serialization. + */ + RecordType(int idx) { + this.idx = idx; + } + + /** + * @return Index for serialization. + */ + public int index() { + return idx; + } /** */ - private static final RecordType[] VALS = RecordType.values(); + private static final RecordType[] VALS; + + static { + RecordType[] recordTypes = RecordType.values(); + + int maxIdx = 0; + for (RecordType recordType : recordTypes) + maxIdx = Math.max(maxIdx, recordType.idx); + + VALS = new RecordType[maxIdx + 1]; + + for (RecordType recordType : recordTypes) + VALS[recordType.idx] = recordType; + } /** */ - public static RecordType fromOrdinal(int ord) { - return ord < 0 || ord >= VALS.length ? null : VALS[ord]; + public static RecordType fromIndex(int idx) { + return idx < 0 || idx >= VALS.length ? null : VALS[idx]; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index e27faa5f02025..193e492b06d3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -56,7 +56,7 @@ * Record V1 serializer. * Stores records in following format: *