标签：
信息: Starting Servlet Engine: Apache Tomcat/7.0.52 initial begin... 2016-08-23 12:33:28,189 WARN [org.apache.hadoop.util.NativeCodeLoader] - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2016-08-23 12:33:29,836 INFO [util.Utils] - Movies data size:3883 2016-08-23 12:33:33,638 INFO [util.Utils] - Users data size:6040 initial end! 八月 23, 2016 12:33:33 下午 org.apache.coyote.AbstractProtocol start这里是初始化的相关打印，初始化使用InitServlet，在里面调用了Utils的init方法，init方法主要初始化了movies变量和userWithRatedMovies变量和allMovieIds变量，各个变量表示意思如下：
<div class="modal fade" id="myModal1" tabindex="-1" role="dialog" aria-labelledby="myModalLabel"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="progress progress-striped active" style="margin-bottom: 0px; height: 25px; border-radius: 5px;"> <div id="progressId" class="progress-bar" style="width: 1%; height: 100%;">0%</div> </div> </div> </div> </div>全部使用bootstrap的基本样式；
// 弹出窗提示程序正在运行
setProgress("progressId", "0%");
// 开启进度条模态框
openModal("myModal1");
// 定时请求任务进度
t=setTimeout("queryTaskProgress(‘"+ret+"‘)",1000);/**
* 设置进度条
* @param id
* @param value
*/
function setProgress(id,value){
$("#"+id).css("width",value);
$("#"+id).html(value);
}
/**
* 开启模态框
* @param id
*/
function openModal(id){
$(‘#‘+id).on(‘show.bs.modal‘, function(){
var $this = $(this);
var $modal_dialog = $this.find(‘.modal-dialog‘);
// 关键代码，如没将modal设置为 block，则$modala_dialog.height() 为零
$this.css(‘display‘, ‘block‘);
$modal_dialog.css({‘margin-top‘: Math.max(0, ($(window).height() - $modal_dialog.height()) / 2) });
});
$(‘#‘+id).modal({backdrop: ‘static‘, keyboard: false});
}/**
* 请求任务进度
*/
function queryTaskProgress(appId){
// ajax 发送请求获取任务运行状态，如果返回运行失败或成功则关闭弹框
$.ajax({
type : "POST",
url : "Monitor",
// dataType : "json",
async:false,// 同步执行
data:{APPID:appId},
success : function(data) {
// console.info("success:"+data);
if(data.indexOf("%")==-1){// 不包含 ，任务运行完成（失败或成功）
clearTimeout(t);// 关闭计时器
// 关闭弹窗进度条
$(‘#myModal1‘).modal("hide");
// 开启提示条模态框
$(‘#tipId‘).html(data=="FINISHED"?"模型训练完成！":
(data=="FAILED"?"调用建模失败!":"模型训练被杀死！"));
openModal("myModal2");
console.info("closed!");
return ;
}
setProgress("progressId", data);
// 进度查询每次间隔1500ms
t=setTimeout("queryTaskProgress(‘"+appId+"‘)",1500);
},
error: function(data){
console.info("error"+data);
}
});
}
String[] runArgs=new String[]{
"--name","ALS Model Train ",
"--class","als.ALSModelTrainer",
"--driver-memory","512m",
"--num-executors", "2",
"--executor-memory", "512m",
"--jar","hdfs://master:8020/user/root/Spark141-als.jar",//
"--files","hdfs://master:8020/user/root/yarn-site.xml",
"--arg",input,
"--arg",output,
"--arg",train_percent,
"--arg",ranks,
"--arg",lambda,
"--arg",iteration
};
FileSystem.get(Utils.getConf()).delete(new Path(output), true);
return Utils.runSpark(runArgs);/**
* 调用Spark 加入监控模块
*
* @param args
* @return Application ID字符串
*/
public static String runSpark(String[] args) {
try {
System.setProperty("SPARK_YARN_MODE", "true");
SparkConf sparkConf = new SparkConf();
sparkConf.set("spark.yarn.jar", "hdfs://master:8020/user/root/spark-assembly-1.4.1-hadoop2.6.0.jar");
sparkConf.set("spark.yarn.scheduler.heartbeat.interval-ms", "1000");
ClientArguments cArgs = new ClientArguments(args, sparkConf);
Client client = new Client(cArgs, getConf(), sparkConf);
// client.run(); // 去掉此种调用方式，改为有监控的调用方式
/**
* 调用Spark ，含有监控
*/
ApplicationId appId = null;
try{
appId = client.submitApplication();
}catch(Throwable e){
e.printStackTrace();
// 返回null
return null;
}
// 开启监控线程
updateAppStatus(appId.toString(),"2%" );// 提交任务完成，返回2%
log.info(allAppStatus.toString());
new Thread(new MonitorThread(appId,client)).start();
return appId.toString();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
// 完成/ 失败/杀死
if (state == YarnApplicationState.FINISHED || state == YarnApplicationState.FAILED
|| state == YarnApplicationState.KILLED) {
Utils.cleanupStagingDir(appId);
// return (state, report.getFinalApplicationStatus);
// 更新 app状态
log.info("Thread:"+Thread.currentThread().getName()+
appId.toString()+"完成，任务状态是："+state.name());
Utils.updateAppStatus(appId.toString(), state.name());
return;
}该代码在MonitorThread中；（但是，需要注意的是，如果Spark任务正在运行，那么这时关闭Tomcat，就会导致相关临时文件删除不了，为什么？请大家自己思考）
<div class="col-md-10 col-md-offset-1" id="movieResultId">接着使用AJax获取后台对应的数据进行拼接，在赋值给div：
// 绑定推荐button
$("#recommendId").click(function(){
var userId = $(‘#userId‘).val();
var recommendNum = $(‘#recommendNumId‘).val();
var ret =null;
$.ajax({
type : "POST",
url : "Recommend",
async:false,// 同步执行
data : {userId:userId,flag:"recommend",recommendNum:recommendNum},
// dataType : "json",
success : function(data) {// data 返回appId
ret = data;
},
error: function(data){
console.info("error"+data);
ret = data=="null"?"null":data;
}
});
var showResultHtml = ‘<br>‘+
‘<p>数据如下：</p>‘+
‘<div class="table-responsive">‘ +
‘<table class="table table-striped">‘ +
‘<thead>‘+
‘<tr>‘+
‘<th>MovieId</th>‘+
‘<th>电影名</th>‘+
‘<th>标签</th>‘+
‘<th>推荐分</th>‘+
‘</tr>‘+
‘</thead>‘+
‘<tbody>‘+
ret +
‘</tbody>‘+
‘</table>‘+
‘</div>‘;
$(‘#movieResultId‘).html(showResultHtml);
});/**
* 预测 如果没有初始化，则进行初始化
*
* @param uid
* @param recNum
* @return
* @throws NoSuchMethodException
* @throws InvocationTargetException
* @throws InstantiationException
* @throws IllegalAccessException
*/
public static List<Movie> predict(int uid,int recNum) throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException {
if (userFeatures.size() <= 0 || productFeatures.size() <= 0) {
try {
userFeatures = getModelFeatures(userFeaturePath);
productFeatures = getModelFeatures(productFeaturePath);
} catch (IOException e) {
return null;
}
if (userFeatures.size() <= 0 || productFeatures.size() <= 0) {
System.err.println("模型加载失败!");
return null;
}
}
// 使用模型进行预测
// 1. 找到uid没有评价过的movieIds
Set<Integer> candidates = Sets.difference((Set<Integer>) allMovieIds, userWithRatedMovies.get(uid));
// 2. 构造推荐排序堆栈
FixSizePriorityQueue<Movie> recommend = new FixSizePriorityQueue<Movie>(recNum);
Movie movie = null;
double[] pFeature = null;
double[] uFeature = userFeatures.get(uid);
double score = 0.0;
BLAS blas = BLAS.getInstance();
for (int candidate : candidates) {
movie = movies.get(candidate).deepCopy();
pFeature = productFeatures.get(candidate);
if (pFeature == null)
continue;
score = blas.ddot(pFeature.length, uFeature, 1, pFeature, 1);
movie.setRated((float) score);
recommend.add(movie);
}
return recommend.sortedList();
}标签：
原文地址：http://blog.csdn.net/fansy1990/article/details/52289826
