Skip to content

Commit 2d2efd6

Browse files
tenfyzhongtenfyzhong
committed
Merge branch 'master' into feat-keyspace-for-api
2 parents c24aa8f + deb1415 commit 2d2efd6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+2295
-111
lines changed

‎Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,12 @@ generate_mock: ## Generate mock code.
129129
generate_mock: tools/bin/mockgen
130130
scripts/generate-mock.sh
131131

132+
build-cdc-with-failpoint: check_failpoint_ctl
133+
build-cdc-with-failpoint: ## Build cdc with failpoint enabled.
134+
$(FAILPOINT_ENABLE)
135+
$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go
136+
$(FAILPOINT_DISABLE)
137+
132138
cdc:
133139
$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc
134140

‎deployments/Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM golang:1.23-alpine as builder
2+
RUN apk add --no-cache git make bash
3+
WORKDIR /go/src/github.com/pingcap/ticdc
4+
COPY . .
5+
ENV CDC_ENABLE_VENDOR=0
6+
RUN make cdc
7+
8+
FROM alpine:3.15
9+
RUN apk add --no-cache tzdata bash curl socat
10+
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc /cdc
11+
EXPOSE 8300
12+
CMD [ "/cdc" ]

‎deployments/dev.Dockerfile

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
FROM golang:1.23-alpine as builder
2+
RUN apk add --no-cache git make bash findutils
3+
WORKDIR /go/src/github.com/pingcap/ticdc
4+
COPY . .
5+
6+
RUN --mount=type=cache,target=/root/.cache/go-build,target=/go/pkg/mod make build-cdc-with-failpoint
7+
8+
FROM alpine:3.15
9+
RUN apk add --no-cache tzdata bash curl socat
10+
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc /cdc
11+
EXPOSE 8300
12+
CMD [ "/cdc" ]
13+

‎dockerfile-local renamed to ‎deployments/dockerfile-local

File renamed without changes.

‎deployments/integration-test.Dockerfile

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Specify the image architecture explicitly,
2+
# otherwise it will not work correctly on other architectures.
3+
FROM amd64/centos:centos7 as downloader
4+
5+
ARG BRANCH
6+
ENV BRANCH=$BRANCH
7+
8+
ARG COMMUNITY
9+
ENV COMMUNITY=$COMMUNITY
10+
11+
ARG VERSION
12+
ENV VERSION=$VERSION
13+
14+
ARG OS
15+
ENV OS=$OS
16+
17+
ARG ARCH
18+
ENV ARCH=$ARCH
19+
20+
USER root
21+
WORKDIR /root/download
22+
23+
# Installing dependencies.
24+
RUN yum install -y \
25+
wget
26+
COPY ./scripts/download-integration-test-binaries.sh .
27+
# Download all binaries into bin dir.
28+
RUN ./download-integration-test-binaries.sh $BRANCH $COMMUNITY $VERSION $OS $ARCH
29+
RUN ls ./bin
30+
31+
# Download go into /usr/local dir.
32+
ENV GOLANG_VERSION 1.23.0
33+
ENV GOLANG_DOWNLOAD_URL https://dl.google.com/go/go$GOLANG_VERSION.linux-amd64.tar.gz
34+
RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
35+
&& tar -C /usr/local -xzf golang.tar.gz \
36+
&& rm golang.tar.gz
37+
38+
FROM amd64/centos:centos7
39+
40+
USER root
41+
WORKDIR /root
42+
43+
# Installing dependencies.
44+
RUN yum install -y \
45+
git \
46+
bash-completion \
47+
wget \
48+
which \
49+
gcc \
50+
make \
51+
curl \
52+
tar \
53+
musl-dev \
54+
sudo \
55+
python3 \
56+
psmisc \
57+
procps
58+
RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
59+
RUN yum install -y epel-release-latest-7.noarch.rpm
60+
RUN yum --enablerepo=epel install -y s3cmd
61+
# Install mysql client.
62+
RUN rpm -ivh https://repo.mysql.com/mysql57-community-release-el7-11.noarch.rpm
63+
# See: https://support.cpanel.net/hc/en-us/articles/4419382481815?input_string=gpg+keys+problem+with+mysql+5.7
64+
RUN rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022
65+
RUN yum install mysql-community-client.x86_64 -y
66+
67+
# install java to run the schema regsitry for the avro case.
68+
RUN yum install -y \
69+
java-1.8.0-openjdk \
70+
java-1.8.0-openjdk-devel
71+
72+
# Copy go form downloader.
73+
COPY --from=downloader /usr/local/go /usr/local/go
74+
ENV GOPATH /go
75+
ENV GOROOT /usr/local/go
76+
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
77+
78+
WORKDIR /go/src/github.com/pingcap/ticdc
79+
COPY . .
80+
81+
RUN --mount=type=cache,target=/root/.cache/go-build,target=/go/pkg/mod make integration_test_build cdc
82+
COPY --from=downloader /root/download/bin/* ./bin/
83+
RUN --mount=type=cache,target=/root/.cache/go-build,target=/go/pkg/mod make check_third_party_binary

‎deployments/kafka-consumer.Dockerfile

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM golang:1.23-alpine AS builder
2+
RUN apk add --no-cache make bash git build-base
3+
WORKDIR /go/src/github.com/pingcap/ticdc
4+
COPY . .
5+
6+
RUN --mount=type=cache,target=/go/pkg/mod go mod download
7+
RUN --mount=type=cache,target=/root/.cache/go-build make kafka_consumer
8+
9+
FROM alpine:3.15
10+
11+
RUN apk update && apk add tzdata curl
12+
13+
ENV TZ=Asia/Shanghai
14+
15+
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc_kafka_consumer /cdc_kafka_consumer
16+
17+
ENTRYPOINT ["tail", "-f", "/dev/null"]

‎deployments/pulsar-integration-test.Dockerfile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM openjdk:17 as jdk_container
2+
3+
FROM hub.pingcap.net/jenkins/centos7_golang-1.23:latest
4+
RUN curl https://archive.apache.org/dist/pulsar/pulsar-3.2.0/apache-pulsar-3.2.0-bin.tar.gz -o pulsar.tar.gz && \
5+
tar -xvf pulsar.tar.gz && \
6+
mv apache-pulsar-3.2.0 pulsar && \
7+
rm pulsar.tar.gz
8+
USER root
9+
RUN yum install -y nc
10+
RUN mv pulsar /usr/local
11+
USER jenkins
12+
COPY --from=jdk_container /usr/java /usr/java
13+
ENV PATH="/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:${PATH}"
14+

‎deployments/storage-consumer.Dockerfile

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM golang:1.23-alpine as builder
2+
RUN apk add --no-cache make bash git
3+
WORKDIR /go/src/github.com/pingcap/ticdc
4+
COPY . .
5+
RUN --mount=type=cache,target=/go/pkg/mod go mod download
6+
RUN --mount=type=cache,target=/root/.cache/go-build make storage_consumer
7+
8+
FROM alpine:3.15
9+
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc_storage_consumer /cdc_storage_consumer
10+
11+
ENTRYPOINT ["tail", "-f", "/dev/null"]

‎dockerfile

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎downstreamadapter/eventcollector/dispatcher_stat.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ type dispatcherStat struct {
115115
// lastEventSeq is the sequence number of the last received DML/DDL/Handshake event.
116116
// It is used to ensure the order of events.
117117
lastEventSeq atomic.Uint64
118-
// lastEventCommitTs is the commitTs of the last received DDL/DML events.
118+
// lastEventCommitTs is the commitTs of the last received DDL/DML/SyncPoint events.
119119
lastEventCommitTs atomic.Uint64
120120
// gotDDLOnTS indicates whether a DDL event was received at the sentCommitTs.
121121
gotDDLOnTs atomic.Bool
@@ -336,6 +336,7 @@ func (d *dispatcherStat) filterAndUpdateEventByCommitTs(event dispatcher.Dispatc
336336
d.gotDDLOnTs.Store(false)
337337
d.gotSyncpointOnTS.Store(false)
338338
}
339+
339340
switch event.GetType() {
340341
case commonEvent.TypeDDLEvent:
341342
d.gotDDLOnTs.Store(true)
@@ -346,7 +347,8 @@ func (d *dispatcherStat) filterAndUpdateEventByCommitTs(event dispatcher.Dispatc
346347
switch event.GetType() {
347348
case commonEvent.TypeDDLEvent,
348349
commonEvent.TypeDMLEvent,
349-
commonEvent.TypeBatchDMLEvent:
350+
commonEvent.TypeBatchDMLEvent,
351+
commonEvent.TypeSyncPointEvent:
350352
d.lastEventCommitTs.Store(event.GetCommitTs())
351353
}
352354

0 commit comments

Comments
 (0)