本文主要介绍使用 Java 来操作 Pulsar,文中所使用到的软件版本:Java 17.0.7(Pulsar 服务使用)、Java 1.8.0_341(客户端使用)、Pulsar 3.3.0、pulsar-client 3.3.0。
1、引入依赖
<dependency><groupId>org.apache.pulsar</groupId><artifactId>pulsar-client</artifactId><version>3.3.0</version> </dependency> <dependency><groupId>org.apache.pulsar</groupId><artifactId>pulsar-client-admin</artifactId><version>3.3.0</version> </dependency>
2、初始化 PulsarClient 和 PulsarAdmin
PulsarClient:
@Before public void before() throws PulsarClientException {client = PulsarClient.builder().serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").build(); }
PulsarAdmin:
@Before public void before() throws PulsarClientException {admin = PulsarAdmin.builder().serviceHttpUrl("http://10.49.196.30:8080,10.49.196.31:8080,10.49.196.32:8080").build(); }
3、消费者
3.1、同步消费
@Test public void sync() throws PulsarClientException {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic") //主题.subscriptionName("my-subscription") //订阅名称.subscriptionType(SubscriptionType.Shared) //订阅模式 .subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("topicName={},value={}", message.getTopicName(), message.getValue());consumer.acknowledge(message);} catch (Exception e) {consumer.negativeAcknowledge(message);}} }
3.2、异步消费
public void async() throws InterruptedException {client.newConsumer(Schema.STRING).topic("my-topic2").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).subscribeAsync().thenAccept(this::receiveAsync);Thread.sleep(1000 * 500); }private void receiveAsync(Consumer<String> consumer) {consumer.receiveAsync().thenAccept(message -> {try {log.info("messageId={},value={}", message.getMessageId(), message.getValue());consumer.acknowledge(message);} catch (Exception e) {consumer.negativeAcknowledge(message);}receiveAsync(consumer);}); }
3.3、使用 MessageListener
@Test public void messageListener() throws Exception {PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").listenerThreads(3).build();MessageListener<String> messageListener = (consumer, msg) -> {try {log.info("messageId={},value={}", msg.getMessageId(), msg.getValue());consumer.acknowledge(msg);} catch (Exception e) {consumer.negativeAcknowledge(msg);}};client.newConsumer(Schema.STRING).topic("my-topic").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).messageListener(messageListener).subscribe();Thread.sleep(1000 * 500); }
3.4、重试信主题
/*** 重试信主题* 处理失败的消息会进入重试信主题,达到最大重试次数后进入死信主题*/ @Test public void retryTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-r").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).enableRetry(true).deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(5).build()).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());//TODO:业务处理,可能发生异常//throw new RuntimeException(); consumer.acknowledge(message);} catch (Exception e) {log.error("", e);consumer.reconsumeLater(message, 5L, TimeUnit.SECONDS);}} }
3.5、死信主题
/*** 死信主题* 由确认超时、负面确认或重试信主题 三种情况消息处理失败后,重试最大次数后消息会进入死信主题*/ @Test public void deadTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-d").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(5).initialSubscriptionName("init-sub").build()).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());//TODO:业务处理,可能发生异常//throw new RuntimeException(); consumer.acknowledge(message);} catch (Exception e) {log.error("", e);consumer.negativeAcknowledge(message);}} }/*** 订阅死信队列,处理其中的消息*/ @Test public void consumerDeadTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-d-my-subscription-DLQ").subscriptionName("init-sub").subscriptionType(SubscriptionType.Shared).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());} catch (Exception e) {log.error("", e);}consumer.acknowledge(message);} }
3.6、完整代码
package com.abc.demo.pulsar;import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.Before; import org.junit.Test;import java.util.concurrent.TimeUnit;@Slf4j public class ConsumerCase {private PulsarClient client;@Beforepublic void before() throws PulsarClientException {client = PulsarClient.builder().serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").build();}@Testpublic void sync() throws PulsarClientException {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic") //主题.subscriptionName("my-subscription") //订阅名称.subscriptionType(SubscriptionType.Shared) //订阅模式 .subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("topicName={},value={}", message.getTopicName(), message.getValue());consumer.acknowledge(message);} catch (Exception e) {consumer.negativeAcknowledge(message);}}}@Testpublic void async() throws InterruptedException {client.newConsumer(Schema.STRING).topic("my-topic2").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).subscribeAsync().thenAccept(this::receiveAsync);Thread.sleep(1000 * 500);}private void receiveAsync(Consumer<String> consumer) {consumer.receiveAsync().thenAccept(message -> {try {log.info("messageId={},value={}", message.getMessageId(), message.getValue());consumer.acknowledge(message);} catch (Exception e) {consumer.negativeAcknowledge(message);}receiveAsync(consumer);});}@Testpublic void messageListener() throws Exception {PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").listenerThreads(3).build();MessageListener<String> messageListener = (consumer, msg) -> {try {log.info("messageId={},value={}", msg.getMessageId(), msg.getValue());consumer.acknowledge(msg);} catch (Exception e) {consumer.negativeAcknowledge(msg);}};client.newConsumer(Schema.STRING).topic("my-topic").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).messageListener(messageListener).subscribe();Thread.sleep(1000 * 500);}/*** 重试信主题* 处理失败的消息会进入重试信主题,达到最大重试次数后进入死信主题*/@Testpublic void retryTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-r").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).enableRetry(true).deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(5).build()).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());//TODO:业务处理,可能发生异常//throw new RuntimeException(); consumer.acknowledge(message);} catch (Exception e) {log.error("", e);consumer.reconsumeLater(message, 5L, TimeUnit.SECONDS);}}}/*** 死信主题* 由确认超时、负面确认或重试信主题 三种情况消息处理失败后,重试最大次数后消息会进入死信主题*/@Testpublic void deadTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-d").subscriptionName("my-subscription").subscriptionType(SubscriptionType.Shared).deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(5).initialSubscriptionName("init-sub").build()).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());//TODO:业务处理,可能发生异常//throw new RuntimeException(); consumer.acknowledge(message);} catch (Exception e) {log.error("", e);consumer.negativeAcknowledge(message);}}}/*** 订阅死信队列,处理其中的消息*/@Testpublic void consumerDeadTopic() throws Exception {Consumer<String> consumer = client.newConsumer(Schema.STRING).topic("my-topic-d-my-subscription-DLQ").subscriptionName("init-sub").subscriptionType(SubscriptionType.Shared).subscribe();while (true) {Message<String> message = consumer.receive();try {log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue());} catch (Exception e) {log.error("", e);}consumer.acknowledge(message);}} }
4、Reader
@Test public void reader() throws Exception {//MessageId messageId = MessageId.fromByteArray(Base64.getDecoder().decode("CBcQBTAA"));MessageId messageId = MessageId.earliest;Reader reader = client.newReader(Schema.STRING).topic("my-topic").startMessageId(messageId).create();while (true) {Message msg = reader.readNext();log.info("messageId={},messageIdBase64={},value={}", msg.getMessageId(), Base64.getEncoder().encodeToString(msg.getMessageId().toByteArray()), msg.getValue());} }
完整代码:
package com.abc.demo.pulsar;import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.Before; import org.junit.Test;import java.util.Base64;@Slf4j public class ReaderCase {private PulsarClient client;@Beforepublic void before() throws PulsarClientException {client = PulsarClient.builder().serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").build();}@Testpublic void reader() throws Exception {//MessageId messageId = MessageId.fromByteArray(Base64.getDecoder().decode("CBcQBTAA"));MessageId messageId = MessageId.earliest;Reader reader = client.newReader(Schema.STRING).topic("my-topic").startMessageId(messageId).create();while (true) {Message msg = reader.readNext();log.info("messageId={},messageIdBase64={},value={}", msg.getMessageId(), Base64.getEncoder().encodeToString(msg.getMessageId().toByteArray()), msg.getValue());}} }
5、生产者
5.1、同步发送
@Test public void sync() throws PulsarClientException {Producer<String> producer = client.newProducer(Schema.STRING).topic("my-topic-d").create();for (int i = 0; i < 3; i++) {MessageId messageId = producer.send(("hello" + i));log.info("messageId={}", messageId);}producer.close(); }
5.2、异步发送
@Test public void async() throws InterruptedException {client.newProducer(Schema.STRING).topic("my-topic2").createAsync().thenAccept(producer -> {for (int i = 0; i < 10; i++) {producer.sendAsync("hello" + i).thenAccept(messageId -> {log.info("messageId={}", messageId);});}});Thread.sleep(1000 * 5); }
5.3、详细设置消息
@Test public void configMessage() throws PulsarClientException {Producer<byte[]> producer = client.newProducer().topic("my-topic").create();MessageId messageId = producer.newMessage(Schema.STRING).key("my-key") //设置消息key.eventTime(System.currentTimeMillis()) //设置事件事件.sequenceId(123) //设置 sequenceId.deliverAfter(1, TimeUnit.MINUTES) //延迟投递消息.property("my-key", "my-value") //自定义属性.value("content").send();log.info("messageId={}", messageId);producer.close(); }
5.4、完整代码
package com.abc.demo.pulsar;import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.After; import org.junit.Before; import org.junit.Test;import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit;@Slf4j public class ProducerCase {private PulsarClient client;@Beforepublic void before() throws PulsarClientException {client = PulsarClient.builder()//.serviceUrl("pulsar://10.49.196.33:6650").serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650").build();}@Afterpublic void after() throws PulsarClientException {client.close();}@Testpublic void sync() throws PulsarClientException {Producer<String> producer = client.newProducer(Schema.STRING).topic("my-topic-d").create();for (int i = 0; i < 3; i++) {MessageId messageId = producer.send(("hello" + i));log.info("messageId={}", messageId);}producer.close();}@Testpublic void async() throws InterruptedException {client.newProducer(Schema.STRING).topic("my-topic2").createAsync().thenAccept(producer -> {for (int i = 0; i < 10; i++) {producer.sendAsync("hello" + i).thenAccept(messageId -> {log.info("messageId={}", messageId);});}});Thread.sleep(1000 * 5);}@Testpublic void configMessage() throws PulsarClientException {Producer<byte[]> producer = client.newProducer().topic("my-topic").create();MessageId messageId = producer.newMessage(Schema.STRING).key("my-key") //设置消息key.eventTime(System.currentTimeMillis()) //设置事件事件.sequenceId(123) //设置 sequenceId.deliverAfter(1, TimeUnit.MINUTES) //延迟投递消息.property("my-key", "my-value") //自定义属性.value("content").send();log.info("messageId={}", messageId);producer.close();} }
6、Admin
6.1、Brokers
6.1.1、列出活动 broker
admin.brokers().getActiveBrokers(clusterName)
6.1.2、列出 broker 的命名空间
admin.brokers().getOwnedNamespaces(cluster,brokerUrl);
6.1.3、获取动态配置名称
admin.brokers().getDynamicConfigurationNames();
6.1.4、更新动态配置
admin.brokers().updateDynamicConfiguration(configName, configValue);
6.1.5、获取已经更新过的动态配置
admin.brokers().getAllDynamicConfigurations();
6.1.6、获取 leader broker
admin.brokers().getLeaderBroker()
6.2、Clusters
6.2.1、获取集群配置信息
admin.clusters().getCluster(clusterName);
6.2.2、获取集群列表
admin.clusters().getClusters();
6.3、Tenant
6.3.1、列出租户
admin.tenants().getTenants();
6.3.2、创建租户
admin.tenants().createTenant(tenantName, tenantInfo);
6.3.3、获取租户配置信息
admin.tenants().getTenantInfo(tenantName);
6.3.4、删除租户
admin.Tenants().deleteTenant(tenantName);
6.3.5、更新租户
admin.tenants().updateTenant(tenantName, tenantInfo);
6.4、Namespaces
6.4.1、创建命名空间
admin.namespaces().createNamespace(namespace);
6.4.2、获取命名空间策略
admin.namespaces().getPolicies(namespace);
6.4.3、列出命名空间
admin.namespaces().getNamespaces(tenant);
6.4.4、删除命名空间
admin.namespaces().deleteNamespace(namespace);
6.5、Topics
6.5.1、查看订阅的待消费消息(不会消费消息)
admin.topics().peekMessages(topic, subName, numMessages);
6.5.2、通过消息 ID 获取消息
admin.topics().getMessageById(topic, ledgerId, entryId);
6.5.3、通过相对于最早或最新消息的位置来查找消息
admin.topics().examineMessage(topic, "latest", 1);
6.5.4、通过时间获取消息的 ID
admin.topics().getMessageIdByTimestamp(topic, timestamp);
6.5.5、跳过特定主题的某个订阅中的若干条未消费消息
admin.topics().skipMessages(topic, subName, numMessages);
6.5.6、跳过特定主题的某个订阅中的所有未消费消息
admin.topics().skipAllMessages(topic, subName);
6.5.7、根据时间重置游标
admin.topics().resetCursor(topic, subName, timestamp);
6.5.8、获取主题所属的 broker
admin.lookups().lookupTopic(topic);
6.5.9、获取分区主题所属的 broker
admin.lookups().lookupPartitionedTopic(topic);
6.5.10、获取主题的订阅信息
admin.topics().getSubscriptions(topic);
6.5.11、获取最新消息的 ID
admin.topics().getLastMessage(topic);
6.5.12、创建非分区主题
admin.topics().createNonPartitionedTopic(topicName);
6.5.13、删除非分区主题
admin.topics().delete(topic);
6.5.14、列出非分区主题
admin.topics().getList(namespace);
6.5.15、获取非分区主题状态
admin.topics().getStats(topic, false /* is precise backlog */);
6.5.16、获取非分区主题内部状态
admin.topics().getInternalStats(topic);
6.5.17、创建分区主题
admin.topics().createPartitionedTopic(topicName, numPartitions);
6.5.18、获取分区主题元数据信息
admin.topics().getPartitionedTopicMetadata(topicName);
6.5.19、更新分区主题的分区数(只能比原来大)
admin.topics().updatePartitionedTopic(topic, numPartitions);
6.5.20、删除分区主题
admin.topics().deletePartitionedTopic(topic);
6.5.21、列出分区主题
admin.topics().getPartitionedTopicList(namespace);
6.5.22、获取分区主题状态
admin.topics().getPartitionedStats(topic, true /* per partition */, false /* is precise backlog */);
6.5.23、获取分区主题内部状态
admin.topics().getPartitionedInternalStats(topic);
6.5.24、创建订阅
admin.topics().createSubscription(topic, subscriptionName, MessageId.latest);
6.5.25、获取订阅
admin.topics().getSubscriptions(topic);
6.5.26、删除订阅
admin.topics().deleteSubscription(topic, subscriptionName);
6.6、完整代码
package com.abc.demo.pulsar;import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.junit.After; import org.junit.Before; import org.junit.Test;import java.util.Collections; import java.util.HashSet;@Slf4j public class AdminCase {private PulsarAdmin admin;@Beforepublic void before() throws PulsarClientException {admin = PulsarAdmin.builder().serviceHttpUrl("http://10.49.196.30:8080,10.49.196.31:8080,10.49.196.32:8080").build();}@Afterpublic void after() {admin.close();}@Testpublic void broker() throws PulsarAdminException {log.info("getActiveBrokers={}", admin.brokers().getActiveBrokers());log.info("getOwnedNamespaces={}", admin.brokers().getOwnedNamespaces("pulsar-cluster-1", "app1:8080"));log.info("getDynamicConfigurationNames={}", admin.brokers().getDynamicConfigurationNames());admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "true");log.info("getAllDynamicConfigurations={}", admin.brokers().getAllDynamicConfigurations());log.info("getLeaderBroker={}", admin.brokers().getLeaderBroker());}@Testpublic void clusters() throws PulsarAdminException {log.info("getCluster={}", admin.clusters().getCluster("pulsar-cluster-1"));log.info("getClusters={}", admin.clusters().getClusters());}@Testpublic void tenants() throws PulsarAdminException {log.info("getTenants={}", admin.tenants().getTenants());TenantInfoImpl tenantInfo = new TenantInfoImpl();tenantInfo.setAdminRoles(new HashSet<>());tenantInfo.setAllowedClusters(Collections.singleton("pulsar-cluster-1"));admin.tenants().createTenant("test-tenant", tenantInfo);log.info("getTenantInfo={}", admin.tenants().getTenantInfo("test-tenant"));admin.tenants().updateTenant("test-tenant", tenantInfo);admin.tenants().deleteTenant("test-tenant");}@Testpublic void namespaces() throws PulsarAdminException {admin.namespaces().createNamespace("public/test-ns");log.info("getPolicies={}", admin.namespaces().getPolicies("public/default"));log.info("getNamespaces={}", admin.namespaces().getNamespaces("public"));admin.namespaces().deleteNamespace("public/test-ns");}@Testpublic void topics() throws PulsarAdminException {log.info("peekMessages={}", admin.topics().peekMessages("persistent://public/default/my-topic", "my-subscription", 3));Message<byte[]> message = admin.topics().getMessageById("persistent://public/default/my-topic", 171, 16);log.info("getMessageById={}", new String(message.getData()));message = admin.topics().examineMessage("persistent://public/default/my-topic", "latest", 1);log.info("examineMessage={}", new String(message.getData()));log.info("getMessageIdByTimestamp={}", admin.topics().getMessageIdByTimestamp("persistent://public/default/my-topic", System.currentTimeMillis()));admin.topics().skipMessages("persistent://public/default/my-topic", "my-subscription", 1);admin.topics().skipAllMessages("persistent://public/default/my-topic", "my-subscription");admin.topics().resetCursor("persistent://public/default/my-topic", "my-subscription", System.currentTimeMillis() - 1000 * 60 * 15);log.info("lookupTopic={}", admin.lookups().lookupTopic("persistent://public/default/my-topic"));log.info("lookupPartitionedTopic={}", admin.lookups().lookupPartitionedTopic("persistent://public/default/my-topic2"));log.info("getSubscriptions={}", admin.topics().getSubscriptions("persistent://public/default/my-topic"));log.info("getLastMessageId={}", admin.topics().getLastMessageId("persistent://public/default/my-topic"));admin.topics().createNonPartitionedTopic("persistent://public/default/test-topic");admin.topics().delete("persistent://public/default/test-topic");log.info("getList={}", admin.topics().getList("public/default"));log.info("getStats={}", admin.topics().getStats("persistent://public/default/my-topic", false));log.info("getInternalStats={}", admin.topics().getInternalStats("persistent://public/default/my-topic"));admin.topics().createPartitionedTopic("persistent://public/default/test-topic-p", 2);log.info("getPartitionedTopicMetadata={}", admin.topics().getPartitionedTopicMetadata("persistent://public/default/test-topic-p"));admin.topics().updatePartitionedTopic("persistent://public/default/test-topic-p", 3);admin.topics().deletePartitionedTopic("persistent://public/default/test-topic-p");log.info("getStats={}", admin.topics().getPartitionedStats("persistent://public/default/my-topic2", false));log.info("getInternalStats={}", admin.topics().getPartitionedInternalStats("persistent://public/default/my-topic2"));admin.topics().createSubscription("persistent://public/default/my-topic", "test-subscription", MessageId.latest);log.info("getStats={}", admin.topics().getSubscriptions("persistent://public/default/my-topic"));admin.topics().deleteSubscription("persistent://public/default/my-topic", "test-subscription");} }
参考:
https://pulsar.apache.org/docs/3.3.x/client-libraries-java/
https://pulsar.apache.org/docs/3.3.x/admin-api-overview/