Preliminary Exploration of Data Heterogeneity in Canal (Tips)
AD |
Canal canal 1 Canal MySQL binlog MySQL Elasticsearch Canal JOIN (buy_id)(shop_id)joinCanal canal binlogIDID2 Canal canal Canal MySQL master(binary log ) binary log eventsshow binlog eventsslavemasterbinary log events(relay log)slave MySQL Canal canal mysql slave mysql slave mysql master dump mysql master dump binary log slave (canal)canal binary log (byte) Canale servercanaljvminstance 1server1.
Canal canal
1
Canal MySQL binlog MySQL Elasticsearch
Canal
JOIN
(buy_id)(shop_id)join
Canal canal binlogIDID
2
Canal canal Canal
MySQL

- master(binary log ) binary log eventsshow binlog events
- slavemasterbinary log events(relay log)
- slave
MySQL Canal

- canal mysql slave mysql slave mysql master dump
- mysql master dump binary log slave (canal)
- canal binary log (byte)
Canale

- servercanaljvm
- instance 1server1..ninstance)
instance
- eventParser (slavemaster)
- eventSink (ParserStore)
- eventStore ()
- metaManager (&)
3 IntelliJ IDEA Canal Demo
Linux canal Canal Demo Debug
canal Canal
canal Demo example

canal client (server)

IDEA Canal Demo
1Canal Server Demo
package com.alibaba.otter.canal.server;import com.alibaba.otter.canal.instance.core.CanalInstance;import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;import com.alibaba.otter.canal.instance.manager.model.Canal;import com.alibaba.otter.canal.instance.manager.model.CanalParameter;import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;import java.net.InetSocketAddress;import java.nio.ByteBuffer;import java.util.Arrays;public class CanalServerTestMain { protected static final String ZK_CLUSTER_ADDRESS = "127.0.0.1:2181"; protected static final String DESTINATION = "example"; protected static final String DETECTING_SQL = "select 1"; protected static final String MYSQL_ADDRESS = "127.0.0.1"; protected static final String USERNAME = "canal"; protected static final String PASSWORD = "canal"; protected static final String FILTER = ".*\\..*"; /** 500s */ protected static final long RUN_TIME = 120 * 1000; private final ByteBuffer header = ByteBuffer.allocate(4); private CanalServerWithNetty nettyServer; public static void main(String[] args) { CanalServerTestMain test = new CanalServerTestMain(); try { test.setUp(); System.out.println("start"); } catch (Throwable e) { e.printStackTrace(); } finally { System.out.println("sleep"); try { Thread.sleep(RUN_TIME); } catch (Throwable ee) { } test.tearDown(); System.out.println("end"); } } public void setUp() { CanalServerWithEmbedded embeddedServer = new CanalServerWithEmbedded(); embeddedServer.setCanalInstanceGenerator(new CanalInstanceGenerator() { public CanalInstance generate(String destination) { Canal canal = buildCanal(); return new CanalInstanceWithManager(canal, FILTER); } }); nettyServer = CanalServerWithNetty.instance(); nettyServer.setEmbeddedServer(embeddedServer); nettyServer.setPort(11111); nettyServer.start(); // instance embeddedServer.start("example"); } public void tearDown() { nettyServer.stop(); } private Canal buildCanal() { Canal canal = new Canal(); canal.setId(1L); canal.setName(DESTINATION); canal.setDesc("test"); CanalParameter parameter = new CanalParameter(); //parameter.setZkClusters(Arrays.asList(ZK_CLUSTER_ADDRESS)); parameter.setMetaMode(CanalParameter.MetaMode.MEMORY); parameter.setHaMode(CanalParameter.HAMode.HEARTBEAT); parameter.setIndexMode(CanalParameter.IndexMode.MEMORY); parameter.setStorageMode(CanalParameter.StorageMode.MEMORY); parameter.setMemoryStorageBufferSize(32 * 1024); parameter.setSourcingType(CanalParameter.SourcingType.MYSQL); parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306), new InetSocketAddress(MYSQL_ADDRESS, 3306))); parameter.setDbUsername(USERNAME); parameter.setDbPassword(PASSWORD); parameter.setSlaveId(1234L); parameter.setDefaultConnectionTimeoutInSeconds(30); parameter.setConnectionCharset("UTF-8"); parameter.setConnectionCharsetNumber((byte) 33); parameter.setReceiveBufferSize(8 * 1024); parameter.setSendBufferSize(8 * 1024); parameter.setDetectingEnable(false); parameter.setDetectingIntervalInSeconds(10); parameter.setDetectingRetryTimes(3); parameter.setDetectingSQL(DETECTING_SQL); canal.setCanalParameter(parameter); return canal; }}
2Canal Client Demo
package com.alibaba.otter.canal.example;import java.net.InetSocketAddress;import java.util.List;import com.alibaba.otter.canal.client.CanalConnectors;import com.alibaba.otter.canal.client.CanalConnector;import com.alibaba.otter.canal.common.utils.AddressUtils;import com.alibaba.otter.canal.protocol.CanalEntry;import com.alibaba.otter.canal.protocol.Message;import com.alibaba.otter.canal.protocol.CanalEntry.Column;import com.alibaba.otter.canal.protocol.CanalEntry.EventType;public class SimpleCanalClientExample { public static void main(String[] args) { // CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(AddressUtils.getHostIp(), 11111), "example", "", ""); int batchSize = 1000; int emptyCount = 0; try { connector.connect(); connector.subscribe(".*..*"); connector.rollback(); int totalEmptyCount = 3000; while (emptyCount < totalEmptyCount) { Message message = connector.getWithoutAck(batchSize); // long batchId = message.getId(); int size = message.getEntries().size(); if (batchId == -1 || size == 0) { emptyCount++; System.out.println("empty count : " + emptyCount); try { Thread.sleep(1000); } catch (InterruptedException e) { } } else { emptyCount = 0; // System.out.printf("message[batchId=%s,size=%s] n", batchId, size); printEntry(message.getEntries()); } connector.ack(batchId); // // connector.rollback(batchId); // , } System.out.println("empty too many times, exit"); } finally { connector.disconnect(); } } private static void printEntry(List<CanalEntry.Entry> entrys) { for (CanalEntry.Entry entry : entrys) { if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { continue; } CanalEntry.RowChange rowChage = null; try { rowChage = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); } catch (Exception e) { throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e); } CanalEntry.EventType eventType = rowChage.getEventType(); System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType)); for (CanalEntry.RowData rowData : rowChage.getRowDatasList()) { if (eventType == CanalEntry.EventType.DELETE) { printColumn(rowData.getBeforeColumnsList()); } else if (eventType == EventType.INSERT) { printColumn(rowData.getAfterColumnsList()); } else { System.out.println("-------> before"); printColumn(rowData.getBeforeColumnsList()); System.out.println("-------> after"); printColumn(rowData.getAfterColumnsList()); } } } } private static void printColumn(List<Column> columns) { for (Column column : columns) { System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated()); } }}
client

binlog

IDEA Demo canal canal
Disclaimer: The content of this article is sourced from the internet. The copyright of the text, images, and other materials belongs to the original author. The platform reprints the materials for the purpose of conveying more information. The content of the article is for reference and learning only, and should not be used for commercial purposes. If it infringes on your legitimate rights and interests, please contact us promptly and we will handle it as soon as possible! We respect copyright and are committed to protecting it. Thank you for sharing.(Email:[email protected])
Mobile advertising space rental |
Tag: Preliminary Exploration of Data Heterogeneity in Canal Tips
Founder of ofo Xiaohuangche, Dai Wei, went to the United States to start a business: registered users drink coffee for free, with an estimated value of $200 million
Next21 rules for dividing databases and tables, hold!
Guess you like
-
Pinduoduo's "Trillion-Yuan Support" Plan: A Three-Year, 100 Billion Yuan Investment to Build a Multi-Win Business EcosystemDetail
2025-04-03 14:41:29 11
-
Huyu Xianxiang and AVIC Optoelectronics Institute Forge Strategic Partnership to Shape China's eVTOL Avionics LandscapeDetail
2025-04-02 18:39:02 1
-
Haier Smart Home's 8th Global R&D Innovation Awards: Illuminating Better Lives with Technology, Achieving User SatisfactionDetail
2025-04-02 15:57:33 21
-
Huawei's 2025 China Digital Power Partner Conference: Carbon-Neutral Path for China, Shared Value CreationDetail
2025-03-31 18:57:09 11
-
OPPO Think Tank: A New Paradigm for Chinese Enterprises' Globalization From Wusha Village to the Global High-End MarketDetail
2025-03-31 18:48:21 11
-
ICLR 2025: Chinese Universities and Companies Showcase AI Prowess with Numerous Accepted Papers; Stanford-HKUST Collaboration Achieves Perfect ScoreDetail
2025-03-31 14:54:45 11
-
Huawei HarmonyOS Smart Home Partner Summit: Deep Dive into Spatial Intelligence Transformation and Ecosystem Development StrategyDetail
2025-03-31 13:01:45 1
-
AI Large Models Drive Innovation in Humanoid Robots and Autonomous Driving: 2025 as a Key MilestoneDetail
2025-03-31 13:00:04 11
-
Eight Cities Pilot Credit Supervision Data Openness, Empowering Micro and Small Enterprises with Mobile Payment PlatformsDetail
2025-03-26 09:32:47 1
-
Xiaomi's "Just a Little Profit": The Deep Logic and Sustainability Behind its Low-Margin StrategyDetail
2025-03-25 15:07:32 21
- Detail
-
The Ninth Huawei ICT Competition China Challenge Finals Conclude Successfully: Kunpeng and Ascend Tracks Crown Their ChampionsDetail
2025-03-24 16:26:03 11
-
Ronshen Sugar Cube Refrigerator: The Official Product of the 2025 FIFA Club World Cup, Ushering in a New Era of Healthy Food PreservationDetail
2025-03-24 15:40:35 21
-
Zhihu Launches New Version of Zhihu Straight Answer: Deep Integration of AI and Community to Enhance Professionalism and CredibilityDetail
2025-03-24 14:04:38 1
-
China Construction Ninth Harmony (Zhongjian Jiuhe) and Huawei HarmonyOS Smart Home Deepen Strategic Partnership at AWE2025, Building a Green and Intelligent Future HomeDetail
2025-03-23 15:21:15 41
-
ZuoYeBang Books Leads the New Trend in Intelligent Education Publishing at Changsha Book FairDetail
2025-03-21 15:15:33 31
-
Tianyancha: Shielding Consumer Safety and Reshaping Business Trust with DataDetail
2025-03-21 08:47:58 1
-
Hisense at AWE2025: AI Empowerment, Leading the Transformation of Future Smart LivingDetail
2025-03-20 18:24:11 11
-
Haier TV Makes a Stunning Debut at AWE 2024: Zhiyuan AI Large Model and PureScene Care Screen Usher in a New Era of Smart HomesDetail
2025-03-20 15:17:20 1
-
China Power's Xin Yuan Zhi Chu (New Source Smart Storage): Open Energy Intelligence Computing Center Leads Intelligent Transformation of the Energy IndustryDetail
2025-03-20 15:15:39 1