1995 offchain voting json

cardano-chain-gen/test/testfiles/invalid-vote-doNotList.jsonld

@@ -0,0 +1,110 @@
+{
+  "@context": {
+    "CIP100": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0100/README.md#",
+    "CIP119": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0119/README.md#",
+    "hashAlgorithm": "CIP100:hashAlgorithm",
+    "body": {
+      "@id": "CIP119:body",
+      "@context": {
+        "references": {
+          "@id": "CIP119:references",
+          "@container": "@set",
+          "@context": {
+            "GovernanceMetadata": "CIP100:GovernanceMetadataReference",
+            "Identity": "CIP119:IdentityReference",
+            "Link": "CIP119:LinkReference",
+            "Other": "CIP100:OtherReference",
+            "label": "CIP100:reference-label",
+            "uri": "CIP100:reference-uri",
+            "referenceHash": {
+              "@id": "CIP119:referenceHash",
+              "@context": {
+                "hashDigest": "CIP119:hashDigest",
+                "hashAlgorithm": "CIP100:hashAlgorithm"
+              }
+            }
+          }
+        },
+        "paymentAddress": "CIP119:paymentAddress",
+        "givenName": "CIP119:givenName",
+        "image": "CIP119:image",
+        "objectives": "CIP119:objectives",
+        "motivations": "CIP119:motivations",
+        "qualifications": "CIP119:qualifications",
+        "doNotList": "CIP119:doNotList"
+      }
+    },
+    "authors": {
+      "@id": "CIP100:authors",
+      "@container": "@set",
+      "@context": {
+        "name": "http://xmlns.com/foaf/0.1/name",
+        "witness": {
+          "@id": "CIP100:witness",
+          "@context": {
+            "witnessAlgorithm": "CIP100:witnessAlgorithm",
+            "publicKey": "CIP100:publicKey",
+            "signature": "CIP100:signature"
+          }
+        }
+      }
+    }
+  },
+  "authors": [],
+  "hashAlgorithm": "blake2b-256",
+  "body": {
+    "doNotList": "false",
+    "givenName": "CaFi",
+    "motivations": "With a strong belief in the potential of blockchain technology and the development of the Cardano ecosystem, the CaFi team wishes to contribute to innovation and progress through supporting community projects and initiatives. The main motivation of CaFi is to make the Catalyst more accessible to Vietnamese, while creating an environment where people can learn, share and develop their skills in writing proposals, evaluating and participating in the democratic voting process.\n\nThe CaFi team understands that the blockchain ecosystem in general and Cardano in particular are in the development stage and require community participation to achieve sustainability and success. By guiding the community to participate in Catalyst, CaFi wants to promote creativity, innovation and help valuable ideas receive the necessary support to grow.",
+    "objectives": "1. Community Support: CaFi team focuses on helping the Vietnam community actively participate in the Catalyst through consulting and training activities on proposal writing, proposal evaluation and voting.\n\n2. Promoting innovation: We wishe to support and develop innovative projects in the Cardano ecosystem, especially those with practical value and sustainable development potential.\n\n3. Building the dRep role: We aim to become a responsible and reputable dRep in Cardano's decentralized governance model, contributing to the long-term development of the ecosystem.\n\n4. Community development: We want to build a dynamic, creative and self-directed community in the Blockchain ecosystem through participating in funding opportunities from Catalyst.\n\nWith the experience gained, CaFi team believes that working with the community will bring many values, while helping projects from Vietnam to have better access to resources from Catalyst to develop strongly on the Cardano platform.",
+    "paymentAddress": "addr1q9mrk85srqxk88239chlxd73ct2s4nl20jzfwgu9k7tzxe60knquflpw8mtql842hyundh3vdy4e6zhw8f6z7afxjvys34qg2p",
+    "qualifications": "1.  Do Viet Cuong\n● Master, high school teacher\n● Blockchain research since 2017\n● Cardano Vietnam community manager\n● Co-host of 9 Catalyst funded proposals\n● Admin of youtube channel, podcast, discussion group about Cardano and Catalyst project\n● Certified CBCA\n\n2. Tu Nguyen\n● Vietnam Drep Pioneer Program #1\n● FIMI Translator team\n● CBCA Alpha Program 2, Certificate ID: 64898843661a2dbd9402d27b\n● CBCA Course, Certificate ID: 5351588\n● Catalyst Funded Proposer\n● Admin of the Catalyst discussion group\n\n3.  Duc Nguyen\n● Cardano Blockchain Researcher from 2018 to present: Research and develop project related to Cardano Blockchain\n● Skill: Financial analysis; IT system management, database; Graphic design\n● Language: Vietnamese; English\n\n4.  Hoang Phuong Linh\n● Translation and research in Cardano Blockchain since 2017\n● Marketer, content creator, trainer, event organizer, community connector\n● Certified CBCA\n● Catalyst Funded Proposer\n● Languages: Vietnamese, English",
+    "references": [
+      {
+        "@type": "Link",
+        "label": "Catalyst Discussion Group",
+        "uri": "https://t.me/Fimi_PA"
+      },
+      {
+        "@type": "Link",
+        "label": "Youtube Channel",
+        "uri": "https://www.youtube.com/@taichinh-taman5516"
+      },
+      {
+        "@type": "Link",
+        "label": "General Cardano Discussion Group",
+        "uri": "https://t.me/StakingADA"
+      },
+      {
+        "@type": "Link",
+        "label": "Cardano Podcast Channel",
+        "uri": "https://t.me/fimi_podcast"
+      },
+      {
+        "@type": "Link",
+        "label": "IOG Reseacher Paper Library in Vietnamese Version",
+        "uri": "https://cardano.vn/docs/Cardano360/TechDocs/intro/"
+      },
+      {
+        "@type": "Identity",
+        "label": "Do Viet Cuong",
+        "uri": "https://t.me/dovietcuong"
+      },
+      {
+        "@type": "Identity",
+        "label": "Tu Nguyen",
+        "uri": "https://t.me/Tulibra"
+      },
+      {
+        "@type": "Identity",
+        "label": "Duc Nguyen",
+        "uri": "https://www.linkedin.com/in/minh-983aa4241/"
+      },
+      {
+        "@type": "Identity",
+        "label": "Hoang Phuong Linh",
+        "uri": "https://www.linkedin.com/in/phuong-linh-hoang-a318bb138/"
+      }
+    ]
+  }
+}

cardano-db-sync/app/http-get-json-metadata.hs

@@ -143,11 +143,12 @@ runGetVote :: Text.Text -> Maybe VoteMetaHash -> DB.AnchorType -> IO ()
 runGetVote file mExpectedHash vtype = do
   respBs <- BS.readFile (Text.unpack file)
   let respLBs = fromStrict respBs
-  (ocvd, val, hsh, mWarning) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing
-  print ocvd
+  (mocvd, val, hsh, mWarning, isValidJson) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing
+  print mocvd
   print val
   print $ bsBase16Encode hsh
   print mWarning
+  putStrLn $ "Is valid JSON: " ++ show isValidJson
 
 -- ------------------------------------------------------------------------------------------------
 

cardano-db-sync/cardano-db-sync.cabal

@@ -334,6 +334,7 @@ test-suite test
                         Cardano.DbSync.Era.Shelley.Generic.ScriptDataTest
                         Cardano.DbSync.Era.Shelley.Generic.ScriptTest
                         Cardano.DbSync.Gen
+                        Cardano.DbSync.OffChain.VoteTest
                         Cardano.DbSync.Util.AddressTest
                         Cardano.DbSync.Util.Bech32Test
                         Cardano.DbSync.Util.CborTest

cardano-db-sync/src/Cardano/DbSync/OffChain.hs

@@ -349,27 +349,71 @@ fetchOffChainVoteData gateways time oVoteWorkQ =
     convert eres =
       case eres of
         Right sVoteData ->
-          let
-            offChainData = sovaOffChainVoteData sVoteData
-            minimalBody = Vote.getMinimalBody offChainData
-            vdt =
-              DB.OffChainVoteData
-                { DB.offChainVoteDataLanguage = Vote.getLanguage offChainData
-                , DB.offChainVoteDataComment = Vote.textValue <$> Vote.comment minimalBody
-                , DB.offChainVoteDataBytes = sovaBytes sVoteData
-                , DB.offChainVoteDataHash = sovaHash sVoteData
-                , DB.offChainVoteDataJson = sovaJson sVoteData
-                , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ
-                , DB.offChainVoteDataWarning = sovaWarning sVoteData
-                , DB.offChainVoteDataIsValid = Nothing
-                }
-            gaF ocvdId = mkGovAction ocvdId offChainData
-            drepF ocvdId = mkDrep ocvdId offChainData
-            authorsF ocvdId = map (mkAuthor ocvdId) $ Vote.getAuthors offChainData
-            referencesF ocvdId = map (mkReference ocvdId) $ mListToList $ Vote.references minimalBody
-            externalUpdatesF ocvdId = map (mkexternalUpdates ocvdId) $ mListToList $ Vote.externalUpdates minimalBody
-           in
-            OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
+          case (sovaIsValidJson sVoteData, sovaOffChainVoteData sVoteData) of
+            -- Scenario 1: Valid JSON + Valid CIP schema
+            (True, Just offChainData) ->
+              let
+                minimalBody = Vote.getMinimalBody offChainData
+                vdt =
+                  DB.OffChainVoteData
+                    { DB.offChainVoteDataLanguage = Vote.getLanguage offChainData
+                    , DB.offChainVoteDataComment = Vote.textValue <$> Vote.comment minimalBody
+                    , DB.offChainVoteDataBytes = sovaBytes sVoteData
+                    , DB.offChainVoteDataHash = sovaHash sVoteData
+                    , DB.offChainVoteDataJson = sovaJson sVoteData
+                    , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ
+                    , DB.offChainVoteDataWarning = sovaWarning sVoteData
+                    , DB.offChainVoteDataIsValid = Just True
+                    }
+                gaF ocvdId = mkGovAction ocvdId offChainData
+                drepF ocvdId = mkDrep ocvdId offChainData
+                authorsF ocvdId = map (mkAuthor ocvdId) $ Vote.getAuthors offChainData
+                referencesF ocvdId = map (mkReference ocvdId) $ mListToList $ Vote.references minimalBody
+                externalUpdatesF ocvdId = map (mkexternalUpdates ocvdId) $ mListToList $ Vote.externalUpdates minimalBody
+               in
+                OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
+            -- Scenario 2: Valid JSON but invalid CIP schema
+            (True, Nothing) ->
+              let
+                vdt =
+                  DB.OffChainVoteData
+                    { DB.offChainVoteDataLanguage = ""
+                    , DB.offChainVoteDataComment = Nothing
+                    , DB.offChainVoteDataBytes = sovaBytes sVoteData
+                    , DB.offChainVoteDataHash = sovaHash sVoteData
+                    , DB.offChainVoteDataJson = sovaJson sVoteData
+                    , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ
+                    , DB.offChainVoteDataWarning = sovaWarning sVoteData
+                    , DB.offChainVoteDataIsValid = Just False
+                    }
+                gaF _ = Nothing
+                drepF _ = Nothing
+                authorsF _ = []
+                referencesF _ = []
+                externalUpdatesF _ = []
+               in
+                OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
+            -- Scenario 3: Invalid JSON (hash matches but content is not parseable as JSON)
+            (False, _) ->
+              let
+                vdt =
+                  DB.OffChainVoteData
+                    { DB.offChainVoteDataLanguage = ""
+                    , DB.offChainVoteDataComment = Nothing
+                    , DB.offChainVoteDataBytes = sovaBytes sVoteData
+                    , DB.offChainVoteDataHash = sovaHash sVoteData
+                    , DB.offChainVoteDataJson = sovaJson sVoteData -- This will be the error message JSON
+                    , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ
+                    , DB.offChainVoteDataWarning = sovaWarning sVoteData
+                    , DB.offChainVoteDataIsValid = Nothing -- NULL for unparseable JSON
+                    }
+                gaF _ = Nothing
+                drepF _ = Nothing
+                authorsF _ = []
+                referencesF _ = []
+                externalUpdatesF _ = []
+               in
+                OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
         Left err ->
           OffChainVoteResultError $
             DB.OffChainVoteFetchError

cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs

@@ -108,36 +108,46 @@ httpGetOffChainVoteDataSingle vurl metaHash anchorType = do
   let req = httpGetBytes manager request 3000000 3000000 url
   httpRes <- handleExceptT (convertHttpException url) req
   (respBS, respLBS, mContentType) <- hoistEither httpRes
-  (ocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)
+  (mocvd, decodedValue, metadataHash, mWarning, isValidJson) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)
   pure $
     SimplifiedOffChainVoteData
       { sovaHash = metadataHash
       , sovaBytes = respBS
       , sovaJson = Text.decodeUtf8 $ LBS.toStrict (Aeson.encode decodedValue)
       , sovaContentType = mContentType
-      , sovaOffChainVoteData = ocvd
+      , sovaOffChainVoteData = mocvd
       , sovaWarning = mWarning
+      , sovaIsValidJson = isValidJson
       }
   where
     url = OffChainVoteUrl vurl
 
-parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text)
+parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Maybe Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text, Bool)
 parseAndValidateVoteData bs lbs metaHash anchorType murl = do
   let metadataHash = Crypto.digest (Proxy :: Proxy Crypto.Blake2b_256) bs
+  -- First check if hash matches - this is critical and must fail if mismatch
   (hsh, mWarning) <- case unVoteMetaHash <$> metaHash of
     Just expectedMetaHashBs
       | metadataHash /= expectedMetaHashBs ->
           left $ OCFErrHashMismatch murl (renderByteArray expectedMetaHashBs) (renderByteArray metadataHash)
     _ -> pure (metadataHash, Nothing)
-  decodedValue <-
+  -- Hash matches, now try to decode as generic JSON
+  -- If this fails, we still want to store the data with is_valid = NULL and an error message
+  (decodedValue, isValidJson) <-
     case Aeson.eitherDecode' @Aeson.Value lbs of
-      Left err -> left $ OCFErrJsonDecodeFail murl (Text.pack err)
-      Right res -> pure res
-  ocvd <-
-    case Vote.eitherDecodeOffChainVoteData lbs anchorType of
-      Left err -> left $ OCFErrJsonDecodeFail murl (Text.pack err)
-      Right res -> pure res
-  pure (ocvd, decodedValue, hsh, mWarning)
+      Left err ->
+        -- Not valid JSON - create an error message object
+        pure (Aeson.object [("error", Aeson.String "Content is not valid JSON. See bytes column for raw data."), ("parse_error", Aeson.String $ Text.pack err)], False)
+      Right res -> pure (res, True)
+  -- Try to decode into strongly-typed vote data structure (only if JSON was valid)
+  -- If this fails (e.g., doNotList is string instead of bool), we still store with is_valid = false
+  let ocvd =
+        if isValidJson
+          then case Vote.eitherDecodeOffChainVoteData lbs anchorType of
+            Left _err -> Nothing -- Don't fail, just return Nothing (will set is_valid = false)
+            Right res -> Just res
+          else Nothing -- Not valid JSON, so can't parse as CIP
+  pure (ocvd, decodedValue, hsh, mWarning, isValidJson)
 
 httpGetBytes ::
   Http.Manager ->

cardano-db-sync/src/Cardano/DbSync/Types.hs

@@ -193,8 +193,9 @@ data SimplifiedOffChainVoteData = SimplifiedOffChainVoteData
   , sovaBytes :: !ByteString
   , sovaJson :: !Text
   , sovaContentType :: !(Maybe ByteString)
-  , sovaOffChainVoteData :: !Vote.OffChainVoteData
+  , sovaOffChainVoteData :: !(Maybe Vote.OffChainVoteData)
   , sovaWarning :: !(Maybe Text)
+  , sovaIsValidJson :: !Bool
   }
 
 data Retry = Retry