optuna/optuna

# Import data into xgb.DMatrix form
dtrain = xgb.DMatrix(X_train,label=y_train)
dtest = xgb.DMatrix(X_test,label=y_test)
#Set parameters
n_splits=5
random_state=42
tree_method='auto'
predictor='cpu_predictor'
eval_metric='aucpr'
grow_policy='depthwise'
booster='gbtree'
base_score=0.5 
objective_ml='binary:logistic'  
verbosity=1
early_stopping_rounds=100
num_boost_round=50000
parallel_jobs=60
#Instantiate the stratified folds
skfolds = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=random_state)       
# define the search space and the objecive function
def objective(trial):
    # Define the search space
    param_sp = {
        'base_score'            : base_score, 
        'booster'               : booster, 
        'colsample_bytree'      : trial.suggest_discrete_uniform('colsample_bytree',0.7,0.85,0.05),
        'learning_rate'         : trial.suggest_loguniform('learning_rate',0.01,0.1),
        'max_depth'             : trial.suggest_int('max_depth', 4,8,1),
        'objective'             : objective_ml, 
        'scale_pos_weight'      : trial.suggest_uniform('scale_pos_weight',1,100),        
        'subsample'             : trial.suggest_discrete_uniform('subsample',0.5,0.85,0.05),      
        'verbosity'             : verbosity, 
        'tree_method'           : tree_method,            
        'predictor'             : predictor, 
        'eval_metric'           : eval_metric,
        'grow_policy'           : grow_policy        
    }
    #Perform Native API cross validation  
    xgb_cv_results=xgb.cv(param_sp,dtrain,stratified=True,folds=skfolds,metrics=eval_metric,
                          num_boost_round=num_boost_round,early_stopping_rounds=early_stopping_rounds,
                          as_pandas=True,verbose_eval=False,seed=random_state,shuffle=True)
    # Set n_estimators as a trial attribute
    trial.set_user_attr("n_estimators", len(xgb_cv_results))
    #Obtain the number of estimators 
    n_estimators=len(xgb_cv_results)
    #Print the params selected in the trial 
    #Create the params set for obtaining the cross validation score 
    params_cv={
    'learning_rate': trial.params['learning_rate'], 
    'subsample': trial.params['subsample'],
    'colsample_bytree': trial.params['colsample_bytree'],
    'max_depth': trial.params['max_depth'],
    'scale_pos_weight': trial.params['scale_pos_weight'],
    'base_score':base_score,
    'booster':booster,
    'objective':objective_ml,
    'verbosity':verbosity,
    'tree_method':tree_method, 
    'predictor':predictor, 
    'eval_metric':eval_metric,
    'grow_policy': grow_policy,    
    # Specific sklearn api variables   
    'random_state':random_state,
    'n_jobs':parallel_jobs,
    'n_estimators':n_estimators,
    'use_label_encoder':False}
    #Instantiate the XGB Estimator 
    xgb_estimator=XGBClassifier(**params_cv)
    #Obtain the cross validation score - to be used by the trial to rate models
    cv_score=cross_val_score(xgb_estimator, X_train, y_train,scoring='f1',cv=skfolds,n_jobs=parallel_jobs).mean()    
    return cv_score
#Create the Study
study = optuna.create_study(study_name='XGB',direction='maximize',sampler=TPESampler(consider_magic_clip=True,seed=random_state,multivariate=True))
# perform the search
n_trials=360
study.optimize(objective, n_trials=n_trials,n_jobs=parallel_jobs)