Publish milestone together with virtual transactions #188

src/main/java/net/helix/pendulum/Pendulum.java

@@ -11,11 +11,7 @@
 import net.helix.pendulum.network.replicator.Replicator;
 import net.helix.pendulum.service.TipsSolidifier;
 import net.helix.pendulum.service.ledger.impl.LedgerServiceImpl;
-import net.helix.pendulum.service.milestone.impl.LatestSolidMilestoneTrackerImpl;
-import net.helix.pendulum.service.milestone.impl.MilestoneServiceImpl;
-import net.helix.pendulum.service.milestone.impl.MilestoneSolidifierImpl;
-import net.helix.pendulum.service.milestone.impl.MilestoneTrackerImpl;
-import net.helix.pendulum.service.milestone.impl.SeenMilestonesRetrieverImpl;
+import net.helix.pendulum.service.milestone.impl.*;
 import net.helix.pendulum.service.snapshot.SnapshotException;
 import net.helix.pendulum.service.snapshot.impl.LocalSnapshotManagerImpl;
 import net.helix.pendulum.service.snapshot.impl.SnapshotProviderImpl;
@@ -106,6 +102,7 @@ public class Pendulum {
     public final MilestoneSolidifierImpl milestoneSolidifier;
     public final CandidateSolidifierImpl candidateSolidifier;
     public final TransactionRequesterWorkerImpl transactionRequesterWorker;
+    public final VirtualTransactionServiceImpl virtualTransactionService;
 
     public final Tangle tangle;
     public final TransactionValidator transactionValidator;
@@ -153,15 +150,15 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException
                 ? new AsyncTransactionPruner()
                 : null;
         transactionRequesterWorker = new TransactionRequesterWorkerImpl();
-
+        virtualTransactionService = new VirtualTransactionServiceImpl();
         // legacy code
         bundleValidator = new BundleValidator();
         tangle = new Tangle();
         tipsViewModel = new TipsViewModel();
         transactionRequester = new TransactionRequester(tangle, snapshotProvider);
         transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration);
         node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel,
-                latestMilestoneTracker, configuration);
+                latestMilestoneTracker, virtualTransactionService, configuration);
         replicator = new Replicator(node, configuration);
         udpReceiver = new UDPReceiver(node, configuration);
         tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration);
@@ -235,6 +232,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx
             localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration);
         }
         milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration);
+        virtualTransactionService.init(tangle, snapshotProvider, transactionValidator);
         validatorManagerService.init(tangle, snapshotProvider, snapshotService, configuration);
         candidateTracker.init(tangle, snapshotProvider, validatorManagerService, candidateSolidifier, configuration);
         latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, candidateTracker, configuration);

src/main/java/net/helix/pendulum/SignedFiles.java

@@ -1,20 +1,16 @@
 package net.helix.pendulum;
 
-import net.helix.pendulum.crypto.Merkle;
 import net.helix.pendulum.crypto.Sha3;
 import net.helix.pendulum.crypto.Sponge;
 import net.helix.pendulum.crypto.SpongeFactory;
 import net.helix.pendulum.crypto.Winternitz;
+import net.helix.pendulum.crypto.merkle.MerkleTree;
+import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl;
 import net.helix.pendulum.model.Hash;
 import org.apache.commons.lang3.ArrayUtils;
 import org.bouncycastle.util.encoders.Hex;
 
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
+import java.io.*;
 import java.util.Arrays;
 
 public class SignedFiles {
@@ -31,7 +27,7 @@ private static boolean validateSignature(String signatureFilename, String public
         byte[] bundle = Winternitz.normalizedBundle(digest);
         byte[] root;
         int i;
-
+        MerkleTree merkle = new MerkleTreeImpl();
         try (InputStream inputStream = SignedFiles.class.getResourceAsStream(signatureFilename);
              BufferedReader reader = new BufferedReader((inputStream == null)
                  ? new FileReader(signatureFilename) : new InputStreamReader(inputStream))) {
@@ -46,7 +42,7 @@ private static boolean validateSignature(String signatureFilename, String public
 
             if ((line = reader.readLine()) != null) {
                 byte[] lineBytes = Hex.decode(line);
-                root = Merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth);
+                root = merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth);
 
             } else {
                 root = Winternitz.address(mode, digests);
@@ -75,7 +71,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio
                 messageBytes = Hash.NULL_HASH.bytes();
             }
             int requiredLength = (int) Math.ceil(messageBytes.length / 32.0) * 32;
-            byte[] finalizedMessage = Merkle.padding(messageBytes, requiredLength);
+            byte[] finalizedMessage = MerkleTree.padding(messageBytes, requiredLength);
             // crypto snapshot message
             sha3.absorb(finalizedMessage, 0, finalizedMessage.length);
             byte[] signature = new byte[Sha3.HASH_LENGTH];

src/main/java/net/helix/pendulum/TransactionValidator.java

@@ -144,9 +144,6 @@ public int getMinWeightMagnitude() {
      */
     private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
         // ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone
-        if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) {
-            return false;
-        }
         if (transactionViewModel.getAttachmentTimestamp() == 0) {
             return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash())
                     || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE;
@@ -155,6 +152,14 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
                 || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS;
     }
 
+    private boolean isTransactionRequested(TransactionViewModel transactionViewModel) {
+        if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) {
+            //todo if is virtual compute it locally
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Runs the following validation checks on a transaction:
      * <ol>
@@ -174,15 +179,17 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
     public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) {
         transactionViewModel.setMetadata();
         transactionViewModel.setAttachmentData();
+
+        if (!transactionViewModel.isVirtual() && isTransactionRequested(transactionViewModel)) {
+            log.debug("Waiting for transaction... " + transactionViewModel.getHash());
+            throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash());
+        }
         if(hasInvalidTimestamp(transactionViewModel)) {
             throw new StaleTimestampException("Invalid transaction timestamp.");
         }
-        for (int i = VALUE_OFFSET + VALUE_USABLE_SIZE; i < VALUE_OFFSET + VALUE_SIZE; i++) { // todo always false.
-            if (transactionViewModel.getBytes()[i] != 0) {
-                throw new IllegalStateException("Invalid transaction value");
-            }
+        if(transactionViewModel.isVirtual()){
+            return;
         }
-
         int weightMagnitude = transactionViewModel.weightMagnitude;
         if((weightMagnitude < minWeightMagnitude)) {
             throw new IllegalStateException("Invalid transaction hash");
@@ -203,7 +210,8 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m
      * @throws RuntimeException if validation fails
      */
     public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude) {
-        TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(SpongeFactory.Mode.S256)));
+        TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, SpongeFactory.Mode.S256);
+
         runValidation(transactionViewModel, minWeightMagnitude);
         return transactionViewModel;
     }

src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java

@@ -0,0 +1,141 @@
+package net.helix.pendulum.controllers;
+
+import net.helix.pendulum.model.BundleHash;
+import net.helix.pendulum.model.Hash;
+import net.helix.pendulum.model.persistables.Bundle;
+import net.helix.pendulum.model.persistables.BundleNonce;
+import net.helix.pendulum.storage.Indexable;
+import net.helix.pendulum.storage.Persistable;
+import net.helix.pendulum.storage.Tangle;
+import net.helix.pendulum.utils.Pair;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+* The BundleViewModel class is an implementation of the <code> HashesViewModel </code> interface.
+* It consists of an <code> Indexable </code> bundle hash and the Bundle model,
+* which contains the set of transaction hashes that are part of the bundle.
+*/
+public class BundleNonceViewModel implements HashesViewModel {
+   private BundleNonce self;
+   private Indexable hash;
+
+   /**
+   * Constructor with bundle hash
+   * @param hash bundle hash
+   */
+   public BundleNonceViewModel(Hash hash) {
+       this.hash = hash;
+   }
+
+   /**
+   * Constructor with bundle hash and related bundle model
+   * @param hashes bundle model
+   * @param hash transaction hash
+   */
+   private BundleNonceViewModel(BundleNonce hashes, Indexable hash) {
+       self = hashes == null || hashes.set == null ? new BundleNonce(): hashes;
+       this.hash = hash;
+   }
+
+   /**
+   * Get the BundleNonceViewModel of a given bundle hash from the database.
+   * @param tangle
+   * @param hash bundle hash
+   * @return <code> BundleNoncefViewModel </code>
+   */
+   public static BundleNonceViewModel load(Tangle tangle, Indexable hash) throws Exception {
+       return new BundleNonceViewModel((BundleNonce) tangle.load(BundleNonce.class, hash), hash);
+   }
+
+   /**
+   * Convert a transaction set hash into the bundle model.
+   * @param hash transaction hash
+   * @param hashToMerge mergable set hash
+   * @return <code> Map.Entry<Indexable, Persistable> </code> map entry of bundle hash and Bundle model
+   */
+   public static Map.Entry<Indexable, Persistable> getEntry(Hash hash, Hash hashToMerge) throws Exception {
+       BundleNonce hashes = new BundleNonce();
+       hashes.set.add(hashToMerge);
+       return new HashMap.SimpleEntry<>(hash, hashes);
+   }
+
+   /**
+   * Store the bundle hash (key) + belonging transaction hashes (value) in the database.
+   * @param tangle
+   * @return <code> boolean </code> success
+   */
+   public boolean store(Tangle tangle) throws Exception {
+       return tangle.save(self, hash);
+   }
+
+   /**
+   * Get the number of transactions belonging to the bundle.
+   * @return <code> int </code> number
+   */
+   public int size() {
+       return self.set.size();
+   }
+
+   /**
+   * Add a transaction hash to the bundle.
+   * @param theHash transaction hash
+   * @return <code> boolean </code> success
+   */
+   public boolean addHash(Hash theHash) {
+       return getHashes().add(theHash);
+   }
+
+   /**
+   * Get the bundle hash / index.
+   * @return <code> Indexable </code> bundle hash
+   */
+   public Indexable getIndex() {
+       return hash;
+   }
+
+   /**
+   * Get the set of transaction hashes belonging to the bundle.
+   * @return <code> Set<Hash> </code> transaction hashes
+   */
+   public Set<Hash> getHashes() {
+       return self.set;
+   }
+
+   /**
+   * Delete the bundle from the database.
+   * @param tangle
+   */
+   @Override
+   public void delete(Tangle tangle) throws Exception {
+       tangle.delete(BundleNonce.class,hash);
+   }
+
+   /**
+   * Get the first bundle from the database.
+   * @param tangle
+   * @return <code> bundleNonceViewModel </code>
+   */
+   public static BundleNonceViewModel first(Tangle tangle) throws Exception {
+       Pair<Indexable, Persistable> bundlePair = tangle.getFirst(Bundle.class, BundleHash.class);
+       if(bundlePair != null && bundlePair.hi != null) {
+           return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low);
+       }
+       return null;
+   }
+
+   /**
+   * Get the next bundle from the database.
+   * @param tangle
+   * @return <code> bundleNonceViewModel </code>
+   */
+   public BundleNonceViewModel next(Tangle tangle) throws Exception {
+       Pair<Indexable, Persistable> bundlePair = tangle.next(Bundle.class, hash);
+       if(bundlePair != null && bundlePair.hi != null) {
+           return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low);
+       }
+       return null;
+   }
+}

src/main/java/net/helix/pendulum/controllers/RoundViewModel.java

@@ -1,7 +1,9 @@
 package net.helix.pendulum.controllers;
 
 import net.helix.pendulum.TransactionValidator;
-import net.helix.pendulum.crypto.Merkle;
+import net.helix.pendulum.crypto.merkle.MerkleOptions;
+import net.helix.pendulum.crypto.merkle.MerkleTree;
+import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl;
 import net.helix.pendulum.model.Hash;
 import net.helix.pendulum.model.HashFactory;
 import net.helix.pendulum.model.IntegerIndex;
@@ -13,14 +15,7 @@
 import net.helix.pendulum.utils.Pair;
 import net.helix.pendulum.utils.Serializer;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
@@ -256,8 +251,9 @@ public static Set<Hash> getMilestoneTrunk(Tangle tangle, TransactionViewModel tr
                 }
             } else {
                 Set<Hash> prevMilestones = prevMilestone.getHashes();
-                List<List<Hash>> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones));
-                if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) {
+                MerkleTree merkle = new MerkleTreeImpl();
+                byte[] merkleTreeRoot = merkle.getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault());
+                if (transaction.getTrunkTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) {
                     //System.out.println("trunk (prev. milestones): ");
                     if (prevMilestones.isEmpty()) {
                         trunk.add(Hash.NULL_HASH);
@@ -286,10 +282,11 @@ public static Set<Hash> getMilestoneBranch(Tangle tangle, TransactionViewModel t
         if (transaction.getCurrentIndex() == transaction.lastIndex()) {
             // tips merkle root
             Set<Hash> confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security);
-            List<List<Hash>> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips));
+            MerkleTree merkle = new MerkleTreeImpl();
+            byte[] root = merkle.getMerkleRoot(new ArrayList<>(confirmedTips), MerkleOptions.getDefault());
             //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString());
             //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString());
-            if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) {
+            if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(root))) {
                 //System.out.println("branch (tips): ");
                 if (confirmedTips.isEmpty()){
                     branch.add(Hash.NULL_HASH);
@@ -309,8 +306,8 @@ public static Set<Hash> getMilestoneBranch(Tangle tangle, TransactionViewModel t
                 }
             } else {
                 Set<Hash> prevMilestones = prevMilestone.getHashes();
-                List<List<Hash>> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones));
-                if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) {
+                byte[] merkleTreeRoot = new MerkleTreeImpl().getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault());
+                if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) {
                     //System.out.println("branch (prev. milestones): ");
                     if (prevMilestones.isEmpty()) {
                         branch.add(Hash.NULL_HASH);
@@ -433,9 +430,8 @@ public Integer index() {
     }
 
     public Hash getMerkleRoot() {
-        List<List<Hash>> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes()));
-        Hash root = merkleTree.get(merkleTree.size()-1).get(0);
-        return root;
+        MerkleTree merkle = new MerkleTreeImpl();
+        return HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new LinkedList<>(getHashes()), MerkleOptions.getDefault()));
     }
 
     /**