Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gray model refactor #12836

Merged
merged 6 commits into from
Nov 13, 2024
Merged

gray model refactor #12836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6c3cbf8
Develop uniformgraypublish (#12429)
shiyiyue1102 Jul 26, 2024
8af465a
[ISSUE#12217] Add the reconciliation mechanism of gray configuration …
MatthewAden Oct 8, 2024
2537139
Merge remote-tracking branch 'origin/develop' into develop-graypublis…
shiyiyue1102 Nov 1, 2024
b10c024
1.delete aggr model
shiyiyue1102 Nov 1, 2024
e91db0c
Frontend changes for #12217.
MatthewAden Nov 9, 2024
d3b6040
Merge pull request #1 from MatthewAden/front-change-for-issue#12217
shiyiyue1102 Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/src/main/java/com/alibaba/nacos/api/common/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class Constants {

public static final String CLIENT_VERSION_KEY = "ClientVersion";

public static final String CLIENT_IP = "ClientIp";

public static final String UNKNOWN_APP = "UnknownApp";

public static final String DEFAULT_DOMAINNAME = "commonconfig.config-host.taobao.com";
Expand Down
26 changes: 14 additions & 12 deletions ...a/com/alibaba/nacos/api/config/remote/request/cluster/ConfigChangeClusterSyncRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@
* @version $Id: ConfigChangeClusterSyncRequest.java, v 0.1 2020年08月11日 4:30 PM liuzunfei Exp $
*/
public class ConfigChangeClusterSyncRequest extends AbstractConfigRequest {

String tag;


long lastModified;

String grayName;

@Deprecated
boolean isBeta;

boolean isBatch;
@Deprecated
String tag;

public boolean isBeta() {
return isBeta;
Expand All @@ -42,14 +44,6 @@ public void setBeta(boolean beta) {
isBeta = beta;
}

public boolean isBatch() {
return isBatch;
}

public void setBatch(boolean batch) {
isBatch = batch;
}

/**
* Getter method for property <tt>tag</tt>.
*
Expand All @@ -68,6 +62,14 @@ public void setTag(String tag) {
this.tag = tag;
}

public String getGrayName() {
return grayName;
}

public void setGrayName(String grayName) {
this.grayName = grayName;
}

/**
* Getter method for property <tt>lastModified</tt>.
*
Expand Down
20 changes: 20 additions & 0 deletions api/src/main/java/com/alibaba/nacos/api/model/v2/ErrorCode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,26 @@ public enum ErrorCode {
*/
PARAMETER_MISMATCH(20009, "parameter mismatch"),

/**
* config gray request error.
*/
CONFIG_GRAY_OVER_MAX_VERSION_COUNT(20010, "config gray version version over max count"),

/**
* config gray tag v2 rule format invalid.
*/
CONFIG_GRAY_RULE_FORMAT_INVALID(20011, "config gray rule format invalid"),

/**
* config gray tag v2 rule version invalid.
*/
CONFIG_GRAY_VERSION_INVALID(20012, "config gray rule version invalid"),

/**
* config gray request error.
*/
CONFIG_GRAY_NAME_UNRECOGNIZED_ERROR(20013, "config gray name not recognized"),

/**
* service name error.
*/
Expand Down
33 changes: 33 additions & 0 deletions api/src/main/java/com/alibaba/nacos/api/remote/request/RequestMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@

import com.alibaba.nacos.api.ability.constant.AbilityKey;
import com.alibaba.nacos.api.ability.constant.AbilityStatus;
import com.alibaba.nacos.api.common.Constants;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
* RequestMeta info.
Expand All @@ -38,6 +40,8 @@ public class RequestMeta {

private Map<String, String> labels = new HashMap<>();

private Map<String, String> appLabels = new HashMap<>();

private Map<String, Boolean> abilityTable;

public AbilityStatus getConnectionAbility(AbilityKey abilityKey) {
Expand Down Expand Up @@ -90,6 +94,35 @@ public Map<String, String> getLabels() {
*/
public void setLabels(Map<String, String> labels) {
this.labels = labels;
extractAppLabels();
}

private void extractAppLabels() {
HashMap<String, String> applabelsMap = new HashMap<String, String>(8) {
{
put(Constants.APPNAME, labels.get(Constants.APPNAME));
put(Constants.CLIENT_VERSION_KEY, clientVersion);
put(Constants.CLIENT_IP, clientIp);
}
};
labels.entrySet().stream().filter(Objects::nonNull).filter(e -> e.getKey().startsWith(Constants.APP_CONN_PREFIX)
&& e.getKey().length() > Constants.APP_CONN_PREFIX.length() && !e.getValue().trim().isEmpty())
.forEach(entry -> {
applabelsMap.putIfAbsent(entry.getKey().substring(Constants.APP_CONN_PREFIX.length()),
entry.getValue());
});
this.appLabels = applabelsMap;
}

/**
* get labels map with filter of starting with prefix #{@link Constants#APP_CONN_PREFIX} and return a new map trim
* the prefix #{@link Constants#APP_CONN_PREFIX}.
*
* @return map of labels.
* @date 2024/2/29
*/
public Map<String, String> getAppLabels() {
return appLabels;
}

/**
Expand Down
1 change: 0 additions & 1 deletion ...m/alibaba/nacos/api/config/remote/request/cluster/ConfigChangeClusterSyncRequestTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ void before() {
configChangeClusterSyncRequest.setTag(TAG);
configChangeClusterSyncRequest.setBeta(Boolean.TRUE);
configChangeClusterSyncRequest.setLastModified(0L);
configChangeClusterSyncRequest.setBatch(false);
configChangeClusterSyncRequest.putAllHeader(HEADERS);
requestId = injectRequestUuId(configChangeClusterSyncRequest);
}
Expand Down
18 changes: 18 additions & 0 deletions config/src/main/java/com/alibaba/nacos/config/server/constant/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,13 +292,31 @@ public class Constants {

public static final String CONFIG_SEARCH_ACCURATE = "accurate";

/**
* Gray rule.
*/
public static final String GRAY_RULE_TYPE = "type";

public static final String GRAY_RULE_EXPR = "expr";

public static final String GRAY_RULE_VERSION = "version";

public static final String GRAY_RULE_PRIORITY = "priority";

/**
* default nacos encode.
*/
public static final String DEFAULT_NACOS_ENCODE = "UTF-8";

public static final String NACOS_PERSIST_ENCODE_KEY = "nacosPersistEncodingKey";

/**
* config publish type.
*/
public static final String FORMAL = "formal";

public static final String GRAY = "gray";

static String getPersistEncode() {
String persistEncode = System.getenv(NACOS_PERSIST_ENCODE_KEY);
if (StringUtils.isBlank(persistEncode)) {
Expand Down
4 changes: 3 additions & 1 deletion config/src/main/java/com/alibaba/nacos/config/server/constant/PropertiesConstant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,7 @@ public class PropertiesConstant {
public static final String DUMP_CHANGE_WORKER_INTERVAL = "dumpChangeWorkerInterval";

public static final String CONFIG_RENTENTION_DAYS = "nacos.config.retention.days";


public static final String GRAY_CAPATIBEL_MODEL = "nacos.config.gray.compatible.model";

}
76 changes: 48 additions & 28 deletions config/src/main/java/com/alibaba/nacos/config/server/controller/ConfigController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,41 +32,47 @@
import com.alibaba.nacos.config.server.model.ConfigAllInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo4Beta;
import com.alibaba.nacos.config.server.model.ConfigInfoGrayWrapper;
import com.alibaba.nacos.config.server.model.ConfigMetadata;
import com.alibaba.nacos.config.server.model.ConfigRequestInfo;
import com.alibaba.nacos.config.server.model.GroupkeyListenserStatus;
import com.alibaba.nacos.config.server.paramcheck.ConfigBlurSearchHttpParamExtractor;
import com.alibaba.nacos.config.server.paramcheck.ConfigDefaultHttpParamExtractor;
import com.alibaba.nacos.config.server.paramcheck.ConfigListenerHttpParamExtractor;
import com.alibaba.nacos.core.paramcheck.ExtractorManager;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.config.server.model.SameConfigPolicy;
import com.alibaba.nacos.config.server.model.SampleResult;
import com.alibaba.nacos.config.server.model.event.ConfigDataChangeEvent;
import com.alibaba.nacos.config.server.model.ConfigRequestInfo;
import com.alibaba.nacos.config.server.model.form.ConfigForm;
import com.alibaba.nacos.config.server.model.gray.BetaGrayRule;
import com.alibaba.nacos.config.server.model.gray.GrayRuleManager;
import com.alibaba.nacos.config.server.monitor.MetricsMonitor;
import com.alibaba.nacos.config.server.paramcheck.ConfigBlurSearchHttpParamExtractor;
import com.alibaba.nacos.config.server.paramcheck.ConfigDefaultHttpParamExtractor;
import com.alibaba.nacos.config.server.paramcheck.ConfigListenerHttpParamExtractor;
import com.alibaba.nacos.config.server.result.code.ResultCodeEnum;
import com.alibaba.nacos.config.server.service.ConfigChangePublisher;
import com.alibaba.nacos.config.server.service.ConfigOperationService;
import com.alibaba.nacos.config.server.service.ConfigSubService;
import com.alibaba.nacos.core.namespace.repository.NamespacePersistService;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoBetaPersistService;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
import com.alibaba.nacos.config.server.service.trace.ConfigTraceService;
import com.alibaba.nacos.config.server.utils.GroupKey;
import com.alibaba.nacos.config.server.utils.MD5Util;
import com.alibaba.nacos.config.server.utils.ParamUtils;
import com.alibaba.nacos.config.server.utils.PropertyUtil;
import com.alibaba.nacos.config.server.utils.RequestUtil;
import com.alibaba.nacos.config.server.utils.TimeUtils;
import com.alibaba.nacos.config.server.utils.YamlParserUtil;
import com.alibaba.nacos.config.server.utils.ZipUtils;
import com.alibaba.nacos.core.control.TpsControl;
import com.alibaba.nacos.core.namespace.repository.NamespacePersistService;
import com.alibaba.nacos.core.paramcheck.ExtractorManager;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.plugin.auth.constant.ActionTypes;
import com.alibaba.nacos.plugin.auth.constant.SignType;
import com.alibaba.nacos.plugin.encryption.handler.EncryptionHandler;
import com.alibaba.nacos.sys.utils.InetUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -122,6 +128,8 @@ public class ConfigController {

private ConfigInfoBetaPersistService configInfoBetaPersistService;

private ConfigInfoGrayPersistService configInfoGrayPersistService;

private NamespacePersistService namespacePersistService;

private final ConfigOperationService configOperationService;
Expand All @@ -130,14 +138,15 @@ public class ConfigController {

public ConfigController(ConfigServletInner inner, ConfigOperationService configOperationService,
ConfigSubService configSubService, ConfigInfoPersistService configInfoPersistService,
NamespacePersistService namespacePersistService,
ConfigInfoBetaPersistService configInfoBetaPersistService) {
NamespacePersistService namespacePersistService, ConfigInfoBetaPersistService configInfoBetaPersistService,
ConfigInfoGrayPersistService configInfoGrayPersistService) {
this.inner = inner;
this.configOperationService = configOperationService;
this.configSubService = configSubService;
this.configInfoPersistService = configInfoPersistService;
this.namespacePersistService = namespacePersistService;
this.configInfoBetaPersistService = configInfoBetaPersistService;
this.configInfoGrayPersistService = configInfoGrayPersistService;
}

/**
Expand Down Expand Up @@ -195,7 +204,6 @@ public Boolean publishConfig(HttpServletRequest request, HttpServletResponse res
configForm.setEffect(effect);
configForm.setType(type);
configForm.setSchema(schema);

if (StringUtils.isBlank(srcUser)) {
configForm.setSrcUser(RequestUtil.getSrcUserName(request));
}
Expand Down Expand Up @@ -308,14 +316,14 @@ public RestResult<Boolean> deleteConfigs(HttpServletRequest request, @RequestPar
String clientIp = RequestUtil.getRemoteIp(request);
String srcUser = RequestUtil.getSrcUserName(request);
final Timestamp time = TimeUtils.getCurrentTime();
List<ConfigInfo> configInfoList = configInfoPersistService.removeConfigInfoByIds(ids, clientIp, srcUser);
List<ConfigAllInfo> configInfoList = configInfoPersistService.removeConfigInfoByIds(ids, clientIp, srcUser);
if (CollectionUtils.isEmpty(configInfoList)) {
return RestResultUtils.success(true);
}
for (ConfigInfo configInfo : configInfoList) {
for (ConfigAllInfo configInfo : configInfoList) {
ConfigChangePublisher.notifyConfigChange(
new ConfigDataChangeEvent(false, configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), time.getTime()));
new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
time.getTime()));

ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), null, time.getTime(), clientIp, ConfigTraceService.PERSISTENCE_EVENT,
Expand Down Expand Up @@ -419,8 +427,8 @@ public Page<ConfigInfo> fuzzySearchConfig(@RequestParam("dataId") String dataId,
@RequestParam("group") String group, @RequestParam(value = "appName", required = false) String appName,
@RequestParam(value = "tenant", required = false, defaultValue = StringUtils.EMPTY) String tenant,
@RequestParam(value = "config_tags", required = false) String configTags,
@RequestParam(value = "types", required = false) String types,
@RequestParam("pageNo") int pageNo, @RequestParam("pageSize") int pageSize) {
@RequestParam(value = "types", required = false) String types, @RequestParam("pageNo") int pageNo,
@RequestParam("pageSize") int pageSize) {
MetricsMonitor.getFuzzySearchMonitor().incrementAndGet();
Map<String, Object> configAdvanceInfo = new HashMap<>(50);
if (StringUtils.isNotBlank(appName)) {
Expand Down Expand Up @@ -458,15 +466,20 @@ public RestResult<Boolean> stopBeta(HttpServletRequest httpServletRequest,
String remoteIp = getRemoteIp(httpServletRequest);
String requestIpApp = RequestUtil.getAppName(httpServletRequest);
try {
configInfoBetaPersistService.removeConfigInfo4Beta(dataId, group, tenant);
configInfoGrayPersistService.removeConfigInfoGray(dataId, group, tenant, BetaGrayRule.TYPE_BETA, remoteIp,
RequestUtil.getSrcUserName(httpServletRequest));
} catch (Throwable e) {
LOGGER.error("remove beta data error", e);
return RestResultUtils.failed(500, false, "remove beta data error");
}
ConfigTraceService.logPersistenceEvent(dataId, group, tenant, requestIpApp, System.currentTimeMillis(),
remoteIp, ConfigTraceService.PERSISTENCE_EVENT_BETA, ConfigTraceService.PERSISTENCE_TYPE_REMOVE, null);

if (PropertyUtil.isGrayCompatibleModel()) {
configInfoBetaPersistService.removeConfigInfo4Beta(dataId, group, tenant);
}
ConfigChangePublisher.notifyConfigChange(
new ConfigDataChangeEvent(true, dataId, group, tenant, System.currentTimeMillis()));
new ConfigDataChangeEvent(dataId, group, tenant, BetaGrayRule.TYPE_BETA, System.currentTimeMillis()));

return RestResultUtils.success("stop beta ok", true);
}
Expand All @@ -485,14 +498,21 @@ public RestResult<ConfigInfo4Beta> queryBeta(@RequestParam(value = "dataId") Str
@RequestParam(value = "group") String group,
@RequestParam(value = "tenant", required = false, defaultValue = StringUtils.EMPTY) String tenant) {
try {
ConfigInfo4Beta ci = configInfoBetaPersistService.findConfigInfo4Beta(dataId, group, tenant);

if (Objects.nonNull(ci)) {
String encryptedDataKey = ci.getEncryptedDataKey();
Pair<String, String> pair = EncryptionHandler.decryptHandler(dataId, encryptedDataKey, ci.getContent());
ci.setContent(pair.getSecond());
ConfigInfo4Beta configInfo4Beta = null;
ConfigInfoGrayWrapper beta4Gray = configInfoGrayPersistService.findConfigInfo4Gray(dataId, group, tenant,
"beta");
if (Objects.nonNull(beta4Gray)) {
String encryptedDataKey = beta4Gray.getEncryptedDataKey();
Pair<String, String> pair = EncryptionHandler.decryptHandler(dataId, encryptedDataKey,
beta4Gray.getContent());
beta4Gray.setContent(pair.getSecond());
configInfo4Beta = new ConfigInfo4Beta();
BeanUtils.copyProperties(beta4Gray, configInfo4Beta);
configInfo4Beta.setBetaIps(
GrayRuleManager.deserializeConfigGrayPersistInfo(beta4Gray.getGrayRule()).getExpr());
}
return RestResultUtils.success("query beta ok", ci);
return RestResultUtils.success("query beta ok", configInfo4Beta);
} catch (Throwable e) {
LOGGER.error("query beta data error", e);
return RestResultUtils.failed("query beta data error");
Expand Down Expand Up @@ -666,8 +686,8 @@ public RestResult<Map<String, Object>> importAndPublishConfig(HttpServletRequest
null, policy);
for (ConfigInfo configInfo : configInfoList) {
ConfigChangePublisher.notifyConfigChange(
new ConfigDataChangeEvent(false, configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), time.getTime()));
new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
time.getTime()));
ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), requestIpApp, time.getTime(), InetUtils.getSelfIP(),
ConfigTraceService.PERSISTENCE_EVENT, ConfigTraceService.PERSISTENCE_TYPE_PUB,
Expand Down Expand Up @@ -918,8 +938,8 @@ public RestResult<Map<String, Object>> cloneConfig(HttpServletRequest request,
srcIp, null, policy);
for (ConfigInfo configInfo : configInfoList4Clone) {
ConfigChangePublisher.notifyConfigChange(
new ConfigDataChangeEvent(false, configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), time.getTime()));
new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
time.getTime()));
ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant(), requestIpApp, time.getTime(), InetUtils.getSelfIP(),
ConfigTraceService.PERSISTENCE_EVENT, ConfigTraceService.PERSISTENCE_TYPE_PUB,
Expand Down
Loading
Loading