diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java index b52b7cb49..3ea9e07f3 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java +++ b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java @@ -8,12 +8,14 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.junit.Ignore; import org.junit.Test; +import org.nd4j.common.resources.Resources; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.RmsProp; import org.nd4j.linalg.lossfunctions.LossFunctions; +import java.nio.file.Files; import java.util.concurrent.CountDownLatch; @Ignore diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java index 63a69b82c..5abd5a253 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java +++ b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java @@ -21,9 +21,9 @@ import lombok.val; import org.apache.commons.lang3.RandomUtils; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.optimize.solvers.accumulation.SmartFancyBlockingQueue; -import org.deeplearning4j.core.util.ThreadUtils; import org.junit.Ignore; import org.junit.Test; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.factory.Nd4j; import java.util.ArrayList; diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml b/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml index 462bebc95..7806bab88 100644 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml +++ b/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml @@ -31,11 +31,6 @@ - - org.deeplearning4j - deeplearning4j-util - ${project.version} - org.nd4j nd4j-api diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/MovingWindowDataSetFetcher.java b/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/MovingWindowDataSetFetcher.java deleted file mode 100644 index e8bee9092..000000000 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/MovingWindowDataSetFetcher.java +++ /dev/null @@ -1,75 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2015-2018 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.datasets.iterator.impl; - -import org.deeplearning4j.util.MovingWindowMatrix; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.fetcher.BaseDataFetcher; -import org.nd4j.common.util.ArrayUtil; - -import java.util.ArrayList; -import java.util.List; - -/** - * - * Moving window data fetcher. Handles rotation of matrices in all directions - * to generate more examples. - * - * - * @author Adam Gibson - */ -public class MovingWindowDataSetFetcher extends BaseDataFetcher { - - private DataSet data; - private int windowRows = 28, windowColumns = 28; - private int cursor = 0; - - public MovingWindowDataSetFetcher(DataSet data, int windowRows, int windowColumns) { - this.data = data; - this.windowRows = windowRows; - this.windowColumns = windowColumns; - List list = data.asList(); - List flipped = new ArrayList<>(); - for (int i = 0; i < list.size(); i++) { - INDArray label = list.get(i).getLabels(); - List windows = - new MovingWindowMatrix(list.get(i).getFeatures(), windowRows, windowColumns, true) - .windows(true); - for (int j = 0; j < windows.size(); j++) { - flipped.add(new DataSet(windows.get(j), label)); - } - flipped.add(list.get(i)); - } - - this.data = DataSet.merge(flipped); - - } - - /** - * Fetches the next dataset. You need to call this - * to get a new dataset, otherwise {@link #next()} - * just returns the last data applyTransformToDestination fetch - * - * @param numExamples the number of examples to fetch - */ - @Override - public void fetch(int numExamples) { - initializeCurrFromList(data.get(ArrayUtil.range(cursor, cursor + numExamples)).asList()); - - } -} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/glove/AbstractCoOccurrences.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/glove/AbstractCoOccurrences.java index 7c9134e2c..969dbaeb9 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/glove/AbstractCoOccurrences.java +++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/glove/AbstractCoOccurrences.java @@ -29,7 +29,7 @@ import org.deeplearning4j.text.sentenceiterator.PrefetchingSentenceIterator; import org.deeplearning4j.text.sentenceiterator.SentenceIterator; import org.deeplearning4j.text.sentenceiterator.SynchronizedSentenceIterator; import org.deeplearning4j.common.util.DL4JFileUtils; -import org.deeplearning4j.core.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; import org.slf4j.Logger; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java index 1f1dce5f7..c007d4b96 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java +++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java @@ -47,7 +47,7 @@ import org.deeplearning4j.text.sentenceiterator.SentenceIterator; import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter; import org.deeplearning4j.text.sentenceiterator.labelaware.LabelAwareSentenceIterator; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.deeplearning4j.core.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java index 3f2d5f216..d31cc51b0 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java +++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java @@ -47,7 +47,7 @@ import org.deeplearning4j.models.word2vec.VocabWord; import org.deeplearning4j.models.word2vec.wordstore.VocabCache; import org.deeplearning4j.models.word2vec.wordstore.VocabConstructor; import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; -import org.deeplearning4j.core.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; import org.nd4j.linalg.api.memory.enums.LearningPolicy; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java index 10f2a4811..fca1288d0 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java +++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java @@ -27,7 +27,7 @@ import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; import org.deeplearning4j.models.word2vec.Huffman; import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; import org.deeplearning4j.text.invertedindex.InvertedIndex; -import org.deeplearning4j.core.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.threadly.concurrent.PriorityScheduler; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java index 8490a1f99..cb1e860c1 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java +++ b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java @@ -18,7 +18,7 @@ package org.deeplearning4j.text.sentenceiterator; import lombok.NonNull; -import org.deeplearning4j.core.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java index ea06a90d9..70cca449e 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java @@ -20,7 +20,7 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.nn.api.Model; import org.deeplearning4j.optimize.api.BaseTrainingListener; -import org.deeplearning4j.util.ThreadUtils; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.api.ndarray.INDArray; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java index f63af7759..146fba9bb 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java @@ -27,8 +27,8 @@ import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostPro import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; import org.deeplearning4j.optimize.solvers.accumulation.encoding.residual.ResidualClippingPostProcessor; import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.AdaptiveThresholdAlgorithm; -import org.deeplearning4j.util.ThreadUtils; import org.nd4j.common.base.Preconditions; +import org.nd4j.common.util.ThreadUtils; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; import org.nd4j.linalg.api.memory.enums.*; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java index 378870bdf..e530b729c 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java @@ -18,6 +18,7 @@ package org.deeplearning4j.optimize.solvers.accumulation; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.util.ThreadUtils; import java.util.Collection; import java.util.Iterator; @@ -28,8 +29,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.deeplearning4j.util.ThreadUtils; - /** * This BlockingQueue implementation is suited only for symmetric gradients updates, and should NOT be used anywhere else. * diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java b/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java new file mode 100644 index 000000000..60313c1d9 --- /dev/null +++ b/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java @@ -0,0 +1,29 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.common.util; + +public class ThreadUtils { + + private ThreadUtils(){ } + + public static void uncheckedSleep(long sleepTimeMs){ + try{ + Thread.sleep(sleepTimeMs); + } catch (InterruptedException e){ } + } + +}