diff --git a/zhuike/src/main/assets/litepal.xml b/zhuike/src/main/assets/litepal.xml index 313d1af..1164b45 100644 --- a/zhuike/src/main/assets/litepal.xml +++ b/zhuike/src/main/assets/litepal.xml @@ -2,7 +2,7 @@ - + diff --git a/zhuike/src/main/java/com/novelbook/android/BookActivity.java b/zhuike/src/main/java/com/novelbook/android/BookActivity.java index 18ff990..94249b1 100644 --- a/zhuike/src/main/java/com/novelbook/android/BookActivity.java +++ b/zhuike/src/main/java/com/novelbook/android/BookActivity.java @@ -536,8 +536,8 @@ public class BookActivity extends Activity_base { nv.save(); mNovel =nv; } else { - - nv.updateAll("novelId=?", novelId+""); + nv.checkAndUpdate(); + // nv.updateAll("novelId=?", novelId+""); // mNovel = Novel.getNovelBySvrId(novelId); } diff --git a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_Shelf.java b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_Shelf.java index c0044e0..23cf4fe 100644 --- a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_Shelf.java +++ b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_Shelf.java @@ -28,6 +28,7 @@ import com.novelbook.android.db.Novel; import com.novelbook.android.filechooser.FileChooserActivity; import com.novelbook.android.netsubscribe.BookSubscribe; +import com.novelbook.android.netutils.HttpMethods; import com.novelbook.android.netutils.NetUtil; import com.novelbook.android.netutils.OnSuccessAndFaultListener; import com.novelbook.android.netutils.OnSuccessAndFaultSub; @@ -41,12 +42,18 @@ import org.json.JSONObject; import org.litepal.LitePal; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import butterknife.BindView; import butterknife.ButterKnife; import butterknife.OnClick; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; public class Fragment_Shelf extends BasicFragment { @@ -88,7 +95,36 @@ public class Fragment_Shelf extends BasicFragment { noveIds+=novel.getNovelId()+","; } } +void test(int maxAge){ + String url = "http://xiaoshuofenxiang.com/api/r/need-update"; + Request request = new Request.Builder().url(url) // .header("User-Agent", "OkHttp Example") + .removeHeader("Pragma") + .header("Cache-Control", "public, max-age=" + maxAge) + .build(); + + if(maxAge==-1){ + request = new Request.Builder().url(url) .build(); + } + + + + HttpMethods.getOkClient().newCall(request).enqueue(new Callback() { + @Override + public void onFailure(Call call, IOException e) { + + } + + @Override + public void onResponse(Call call, Response response) throws IOException { + ResponseBody body = response.body(); + Log.d(TAG, "onResponse:test " + body.string()); + handler.sendEmptyMessage(1); + } + }); + + +} @Override protected int getLayoutRes() { @@ -99,6 +135,35 @@ public class Fragment_Shelf extends BasicFragment { * to get updated info from server,TODO: put it in service ,scheduled */ private void getUpdatedData(){ + /* try { + test(3600); + Thread.sleep(1000); + + test(3600); + Thread.sleep(1000); + + test(0); + Thread.sleep(1000); + + test(3600); + Thread.sleep(1000); + + test(0); + Thread.sleep(1000); + + test(-1); + Thread.sleep(1000); + + test(3600); + Thread.sleep(1000); + + test(-1); + Thread.sleep(1000); + + } catch (InterruptedException e) { + e.printStackTrace(); + }*/ + List novelIds; OnSuccessAndFaultListener successAndFaultListener = new OnSuccessAndFaultListener() { @Override @@ -110,7 +175,7 @@ public class Fragment_Shelf extends BasicFragment { lstUpdate = GsonUtil. parserJsonArray(result, Constants.BLOCK_TITLE_NOVELS); if(lstUpdate.size()>0) { for (Novel novel2 : lstUpdate) { - novel2.checkAndUpdateShelf(); + novel2.checkAndUpdate(); } loadNovelsOnShelf(); } diff --git a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bangdan.java b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bangdan.java index 351768a..44b8287 100644 --- a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bangdan.java +++ b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bangdan.java @@ -40,14 +40,7 @@ public class Fragment_bangdan extends BasicFragment { RecyclerView rvBandan; // @BindView(R.id.topic_layout) // MultiLayout topic_layout; - private static final String[] TAB_NAMES = { - "全部", "赛事", "原创", "少年", "少女", "日漫", - "杂志", "热血", "搞笑", "治愈", "惊秫", "古风", - "全部全部", "赛事2", "原创1", "少年2", "少女1", "日漫2", - "杂志eeee3", "热血2", "搞笑1", "治愈2", "惊秫2", "古风2", - "全部1", "赛事1", "原创2", "少年1", "少女", "日漫2", - "杂志1", "热血1", "搞笑2", "治愈1", "惊秫1", "古风1" - }; + public static Fragment_bangdan newInstance(String param1, String param2) { Fragment_bangdan fragment = new Fragment_bangdan(); Bundle args = new Bundle(); diff --git a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bookStore.java b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bookStore.java index 562a0ff..90f11ff 100644 --- a/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bookStore.java +++ b/zhuike/src/main/java/com/novelbook/android/Fragments/Fragment_bookStore.java @@ -72,15 +72,17 @@ public class Fragment_bookStore extends BasicFragment { public void initData() { mTitle = new String[]{"精选", "榜单","分类"}; - if(mAdapter!=null) { - for(BasicFragment bf:mFragments){ - bf.initData(); - } - // mViewpager.setCurrentItem(0); - mAdapter.notifyDataSetChanged(); - - } } + public void onSexSwitch(){ + if(mAdapter!=null) { + for(BasicFragment bf:mFragments){ + bf.initData(); + } + mAdapter.notifyDataSetChanged(); + + } + } + @Override public void initViews(){ initTabs(); diff --git a/zhuike/src/main/java/com/novelbook/android/Main2Activity.java b/zhuike/src/main/java/com/novelbook/android/Main2Activity.java index 9aa572d..c0c2eee 100644 --- a/zhuike/src/main/java/com/novelbook/android/Main2Activity.java +++ b/zhuike/src/main/java/com/novelbook/android/Main2Activity.java @@ -465,7 +465,7 @@ private int bottomSelectedIndex; Fragment current = getSupportFragmentManager().findFragmentById(R.id.realtabcontent); if (current != null && current instanceof BasicFragment){ - ((BasicFragment)current).reload(); + ((Fragment_bookStore)current).onSexSwitch(); } @@ -517,7 +517,7 @@ private int bottomSelectedIndex; //----get master domain - private void getHostPolicy(){ + private void getHostPolicy(){ //TODO: get masterdomain info BookSubscribe.getMastDomain(new OnSuccessAndFaultSub(new OnSuccessAndFaultListener() { @Override public void onSuccess(String result) { @@ -543,33 +543,5 @@ private int bottomSelectedIndex; } }, this)); } - void getSearchTabTtitle(){ - BookSubscribe.getSearchTitles(new OnSuccessAndFaultSub(new OnSuccessAndFaultListener() { - @Override - public void onSuccess(String result) { - // mFirstPage= gson.fromJson(result, FirstPage.class); - try { - - Constants.lstSex = GsonUtil.parserStringBlocks(result,"sex"); - Constants.lstNt =GsonUtil.parserStringBlocks(result,"nt"); - // Constants.lstProgress =GsonUtil.parserProgressBlocks(result,"progress"); - - } catch (Exception e) { - e.printStackTrace(); - } - - - } - - @Override - public void onFault(String errorMsg) { - //失败 - Log.d(TAG, "error on get firstpage: " + errorMsg); - - - } - },this)); - - } } diff --git a/zhuike/src/main/java/com/novelbook/android/bean/Regex.java b/zhuike/src/main/java/com/novelbook/android/bean/Regex.java new file mode 100644 index 0000000..40757ed --- /dev/null +++ b/zhuike/src/main/java/com/novelbook/android/bean/Regex.java @@ -0,0 +1,9 @@ +package com.novelbook.android.bean; + +public class Regex { + private String regex; + private String name; + private boolean group; + private String[] output; + private RegexChild child; +} diff --git a/zhuike/src/main/java/com/novelbook/android/bean/RegexChild.java b/zhuike/src/main/java/com/novelbook/android/bean/RegexChild.java new file mode 100644 index 0000000..90c81e2 --- /dev/null +++ b/zhuike/src/main/java/com/novelbook/android/bean/RegexChild.java @@ -0,0 +1,8 @@ +package com.novelbook.android.bean; + +public class RegexChild { + private String regex; + private String name; + private boolean group; + private String[] output; +} diff --git a/zhuike/src/main/java/com/novelbook/android/bean/RegexOnMulu.java b/zhuike/src/main/java/com/novelbook/android/bean/RegexOnMulu.java new file mode 100644 index 0000000..f8a817b --- /dev/null +++ b/zhuike/src/main/java/com/novelbook/android/bean/RegexOnMulu.java @@ -0,0 +1,7 @@ +package com.novelbook.android.bean; + +public class RegexOnMulu { + private String source; + private Regex[] regexes; +} + diff --git a/zhuike/src/main/java/com/novelbook/android/db/Novel.java b/zhuike/src/main/java/com/novelbook/android/db/Novel.java index c92119d..9295897 100644 --- a/zhuike/src/main/java/com/novelbook/android/db/Novel.java +++ b/zhuike/src/main/java/com/novelbook/android/db/Novel.java @@ -3,6 +3,8 @@ package com.novelbook.android.db; import android.text.TextUtils; +import com.novelbook.android.utils.Constants; + import org.litepal.LitePal; import org.litepal.annotation.Column; import org.litepal.crud.LitePalSupport; @@ -309,11 +311,21 @@ public class Novel extends LitePalSupport implements Serializable{ return super.update(id); } - public int checkAndUpdateShelf(){ + public int checkAndUpdate(){ Novel nv = getNovelBySvrId(novelId); - if(lastUpdateTime > nv.lastVisit){ + if(lastUpdateTime > nv.lastVisit || lastUpdateTime > nv.lastUpdateTime){ isUpdated =true; } return super.update(nv.id); } + public int getMaxAge(){ + int maxAge = Constants.MAXAGE_MULU; + if( isFinished){ + maxAge =Constants.MAXAGE_MAX; + }else if(isUpdated){ + maxAge = 0; + } + return maxAge; // return 0;// maxAge; + } + } diff --git a/zhuike/src/main/java/com/novelbook/android/db/SiteRule.java b/zhuike/src/main/java/com/novelbook/android/db/SiteRule.java index 294e666..c0079e2 100644 --- a/zhuike/src/main/java/com/novelbook/android/db/SiteRule.java +++ b/zhuike/src/main/java/com/novelbook/android/db/SiteRule.java @@ -1,5 +1,7 @@ package com.novelbook.android.db; +import com.novelbook.android.bean.RegexOnMulu; + import org.litepal.LitePal; import org.litepal.annotation.Column; import org.litepal.crud.LitePalSupport; @@ -11,7 +13,8 @@ public class SiteRule extends LitePalSupport { private String name; @Column(unique = true, nullable = false) private String domain; - private String chapterUrlRegexOnMulu; + // private String chapterUrlRegexOnMulu; + private RegexOnMulu[] chapterUrlRegexOnMulu; private String chapterContentDumpRegex; private String chapterContentRegex; private String muluUrlPattern; @@ -35,6 +38,14 @@ public class SiteRule extends LitePalSupport { this.chapterContentRegex = chapterContentRegex; } + public RegexOnMulu[] getChapterUrlRegexOnMulu() { + return chapterUrlRegexOnMulu; + } + + public void setChapterUrlRegexOnMulu(RegexOnMulu[] chapterUrlRegexOnMulu) { + this.chapterUrlRegexOnMulu = chapterUrlRegexOnMulu; + } + public String getName() { return name; } @@ -51,14 +62,14 @@ public class SiteRule extends LitePalSupport { this.domain = domain; } - public String getChapterUrlRegexOnMulu() { + /* public String getChapterUrlRegexOnMulu() { return chapterUrlRegexOnMulu; } public void setChapterUrlRegexOnMulu(String chapterUrlRegexOnMulu) { this.chapterUrlRegexOnMulu = chapterUrlRegexOnMulu; } - +*/ public String getChapterContentDumpRegex() { return chapterContentDumpRegex; } diff --git a/zhuike/src/main/java/com/novelbook/android/netutils/HttpMethods.java b/zhuike/src/main/java/com/novelbook/android/netutils/HttpMethods.java index 59147c5..94e2890 100644 --- a/zhuike/src/main/java/com/novelbook/android/netutils/HttpMethods.java +++ b/zhuike/src/main/java/com/novelbook/android/netutils/HttpMethods.java @@ -129,7 +129,7 @@ public class HttpMethods { */ File cacheFile = new File(FileUtils.getDiskCacheDir(MyApp.applicationContext), CACHE_NAME); Cache cache = new Cache(cacheFile, 1024 * 1024 * 50); - + Log.d(TAG, "getClient: to set cach control"); // okHttpBuilder.cache(cache).addInterceptor(cacheInterceptor); okHttpBuilder.cache(cache) .addNetworkInterceptor(REWRITE_RESPONSE_INTERCEPTOR) .addInterceptor(REWRITE_RESPONSE_INTERCEPTOR_OFFLINE); @@ -149,7 +149,7 @@ public class HttpMethods { .method(originalRequest.method(), originalRequest.body()); */ Request.Builder requestBuilder = originalRequest.newBuilder() - // .addHeader("Accept-Encoding", "gzip") + // .addHeader("Accept-Encoding", "gzip") .addHeader("Accept-Encoding", Locale.getDefault().toString() ) // .addHeader("Accept", "application/json") // .addHeader("Content-Type", "application/json; charset=utf-8") @@ -213,6 +213,7 @@ public class HttpMethods { Response originalResponse = chain.proceed(chain.request()); String cacheControl = originalResponse.header("Cache-Control"); + Log.d(TAG, String.format("intercept:url %s, get cache controle: %s, maxAgeForce %s",originalResponse.request().url(), cacheControl,maxAgeForce)); //int maxAge = 60*60; //一小时 if (maxAgeForce >0 ||cacheControl == null || cacheControl.contains("no-store") || cacheControl.contains("no-cache") || cacheControl.contains("must-revalidate") || cacheControl.contains("max-age=0")) { //目标网站禁用cache则设置为1小时 @@ -224,11 +225,11 @@ public class HttpMethods { .header("Cache-Control", "public, max-age=" + age) .build(); } else { - return originalResponse.newBuilder() + /* return originalResponse.newBuilder() .removeHeader("Pragma") .header("Cache-Control", "public, max-age=" + maxAge) - .build(); - // return originalResponse; + .build();*/ + return originalResponse; } } }; diff --git a/zhuike/src/main/java/com/novelbook/android/netutils/OnSuccessAndFaultSub.java b/zhuike/src/main/java/com/novelbook/android/netutils/OnSuccessAndFaultSub.java index 5b1e100..3b84fff 100644 --- a/zhuike/src/main/java/com/novelbook/android/netutils/OnSuccessAndFaultSub.java +++ b/zhuike/src/main/java/com/novelbook/android/netutils/OnSuccessAndFaultSub.java @@ -137,6 +137,10 @@ public class OnSuccessAndFaultSub extends DisposableObserver } else if(code == 500) { // Toast.makeText(context,"网关错误 ，请检查您的网络状态",Toast.LENGTH_LONG).show(); mOnSuccessAndFaultListener.onFault("服务错误"); + } else if(code == 503) { + // Toast.makeText(context,"网关错误 ，请检查您的网络状态",Toast.LENGTH_LONG).show(); + //sleep 200 + mOnSuccessAndFaultListener.onFault("服务限制"); }else if(code == 502) { HttpMethods.getInstance().changeBaseUrl(); //更新基本信息 try { @@ -192,8 +196,9 @@ public class OnSuccessAndFaultSub extends DisposableObserver Log.e("OnSuccessAndFaultSub", "errorMsg: " + errorMsg); } } catch (Exception e) { - Toast.makeText(context,"出错了" + e.getMessage(),Toast.LENGTH_LONG); + Log.e(TAG, "prepare book error: ",e ); e.printStackTrace(); + // Toast.makeText(context,"出错了" + e.getMessage(),Toast.LENGTH_LONG); } } diff --git a/zhuike/src/main/java/com/novelbook/android/utils/BookUtil.java b/zhuike/src/main/java/com/novelbook/android/utils/BookUtil.java index f9dd074..f711d6f 100644 --- a/zhuike/src/main/java/com/novelbook/android/utils/BookUtil.java +++ b/zhuike/src/main/java/com/novelbook/android/utils/BookUtil.java @@ -7,14 +7,12 @@ import android.content.Context; import android.os.AsyncTask; import android.os.Handler; -import android.os.Looper; import android.os.Message; import android.text.TextUtils; import android.util.Log; import android.widget.Toast; import com.google.gson.Gson; -import com.novelbook.android.BookActivity; import com.novelbook.android.MyApp; import com.novelbook.android.bean.Cache; import com.novelbook.android.bean.NovelSites; @@ -33,7 +31,6 @@ import org.json.JSONException; import org.json.JSONObject; import org.litepal.LitePal; -import java.io.Console; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -260,8 +257,15 @@ public class BookUtil { BookSubscribe.getSiteRule(mSite.getDomain(),new OnSuccessAndFaultSub(new OnSuccessAndFaultListener() { @Override public void onSuccess(String result) { + + Log.d(TAG, "prepare book siteRule:" +result); //成功 - SiteRule sr = (SiteRule)gson.fromJson(result,SiteRule.class); + SiteRule sr= new SiteRule(); + try { + sr = (SiteRule) gson.fromJson(result, SiteRule.class); + }catch (Exception e){ + Log.e(TAG, "prepare book error on gson: ", e); + } List srs = LitePal.where("domain=?",sr.getDomain()).limit(1).find(SiteRule.class); long id = srs.size()==1 ?srs.get(0).getId() :0; @@ -274,13 +278,53 @@ public class BookUtil { } mSiteRule =sr; setSiteInfo(); - Log.d(TAG, String.format("prepare book %s 目录正则表达式下载完成,开始读章节信息. thread %s ",mNovel.getName(),Thread.currentThread().getName()) ); + Log.d(TAG, String.format("prepare book %s 目录正则表达式下载完成,开始读章节信息. muluRegex size %s, thread %s ",mNovel.getName(),mSiteRule.getChapterUrlRegexOnMulu().length, Thread.currentThread().getName()) ); Log.d(TAG, String.format("目录正则表达式下载完成，开始读取章节信息") ); - readChaptersAsync(); + if(mSiteRule.getChapterUrlRegexOnMulu().length>0) { + mMuluStatus = MuluStatus.isDownloading; + long startTime= new Date().getTime(); + Log.d(TAG,String.format("prepare book loadChapts----start download %s,maxAge %s, 目录 from %s", mNovel.getName() ,mNovel.getMaxAge() ,mSite.getMuluUrl() )); + new Thread(){ + @Override + public void run() { + Log.d(TAG, "changing Source: to get site rule"); + String[] chaps = new String[0]; + try { + JSONObject siteJson = new JSONObject(result); + + chaps = NovelParseUtil.getChapters(mSite.getMuluUrl(), siteJson); + + if (chaps != null) + for (String s : chaps) { + Log.d(TAG, "prepare book readChaptersAsync: chapt: " + s); + } + + } catch (JSONException e) { + Log.e(TAG, "prepare book error on parese :", e); + } + + if (chaps == null || chaps.length == 0) { + readChaptersAsync(); + }else + { + + Log.d(TAG,String.format("prepare book loadChapts----end download %s 目录, 目录数量 %s, cost %s", mNovel.getName() , mChapters.size(), new Date().getTime() -startTime )); + mMuluStatus = MuluStatus.isDone; + Log.d(TAG, String.format("prepare book %s 章节信息完成.",mNovel.getName()) ); + } + + + } + }.start(); + + + }else{ + readChaptersAsync(); + } } @Override @@ -559,15 +603,7 @@ public class BookUtil { } } -private int getMaxAge(){ - int maxAge = Constants.MAXAGE_MULU; - if(mNovel.isFinished()){ - maxAge =Constants.MAXAGE_MAX; - }else if(mNovel.isUpdated()){ - maxAge = 0; - } - return maxAge; -} + int muluRetryCount =0; @@ -577,10 +613,13 @@ int muluRetryCount =0; return; } String url = mSite.getMuluUrl(); - Request request = getTagRequest(url); + Request request = getTagRequest(url,mNovel.getMaxAge()); + mMuluStatus = MuluStatus.isDownloading; long startTime= new Date().getTime(); - Log.d(TAG,String.format("prepare book loadChapts----start download %s,maxAge %s, 目录 from %s", mNovel.getName() ,getMaxAge() ,url )); + Log.d(TAG,String.format("prepare book loadChapts----start download %s,maxAge %s, 目录 from %s", mNovel.getName() ,mNovel.getMaxAge() ,url )); + + /* if(muluRetryCount<3){ muluRetryCount++; @@ -591,7 +630,7 @@ int muluRetryCount =0; return; }*/ - HttpMethods.getOkClient(getMaxAge()).newCall(request).enqueue(new Callback() { + HttpMethods.getOkClient().newCall(request).enqueue(new Callback() { @Override public void onFailure(Call call, IOException e) { // Log.d(TAG, "onFailure: " + e.getMessage()); @@ -645,8 +684,6 @@ int muluRetryCount =0; public void onResponse(Call call, Response response){ ResponseBody body = response.body(); - - if(response.code()!=200){ Log.d(TAG,String.format("prepare book loadChapts----failed, %s 目录 from %s,return code %s", mNovel.getName() ,url,response.code() )); handler.sendEmptyMessage(3); @@ -685,6 +722,8 @@ int muluRetryCount =0; }); } void buildChapters( String content ,String url){ + + Log.d(TAG, "buildChapters: " +content); if(mSite==null ||mSiteRule ==null){ Log.d(TAG,String.format("prepare book buildChapters failed---- %s ,mSite is null? %s ,mSiteRule ==null ? %s", mNovel.getName() ,mSite==null,mSiteRule ==null )); return; @@ -698,7 +737,7 @@ int muluRetryCount =0; }else{ siteJson.put("chapterUrlRegexOnMulu", mSiteRule.getChapterUrlRegexOnMulu()); }*/ - siteJson.put("chapterUrlRegexOnMulu", mSiteRule.getChapterUrlRegexOnMulu()); + siteJson.put("chapterUrlRegexOnMulu", ""); // mChapters = NovelParseUtil.getChapters(mSite.getDomain(),url, content, siteJson); @@ -710,7 +749,7 @@ int muluRetryCount =0; } catch (JSONException e) { // } catch (JSONException | IOException e) { - Log.d(TAG,String.format("prepare book, mulu on Site %s download status %s",mSite.getDomain(),mMuluStatus)); + Log.e(TAG,String.format("prepare book, mulu on Site %s download status %s",mSite.getDomain(),mMuluStatus),e); e.printStackTrace(); } finally { // result.close(); @@ -1411,7 +1450,7 @@ private void loadChaptContent(final int chapterIndex) throws JSONException, Inte JSONObject siteJson = new JSONObject(); siteJson.put("chapterContentRegex", mSiteRule.getChapterContentRegex()); siteJson.put("chapterContentDumpRegex", mSiteRule.getChapterContentDumpRegex()); - Request request = getTagRequest(url); + Request request = getTagRequest(url, 0); HttpMethods.getOkClient().newCall(request).enqueue(new Callback() { @Override public void onFailure(Call call, IOException e) { @@ -1490,12 +1529,27 @@ private void loadChaptContent(final int chapterIndex) throws JSONException, Inte } - private Request getTagRequest(String url) { - return new Request.Builder() + /*** + * + * @param url + * @param maxAge + * @return + */ + private Request getTagRequest(String url, int maxAge) { + + Request.Builder builder = new Request.Builder() .tag(mNovel.getNovelId()) //标记 请求的tag,切换小说或离开小说界面(BookActivity) 时 取消未执行完毕的 此tag的所有请求 .url(url) - // .header("User-Agent", "OkHttp Example") - .build(); + .removeHeader("Pragma") + .header("Cache-Control", "public, max-age=" + 0) + // .header("Accept-Encoding","gzip, deflate, sdch") + .header("Accept-Language","zh-CN,zh;q=0.9") + .header( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") + .header( "Upgrade-Insecure-Requests", "1") + ; + + return builder.build(); + } diff --git a/zhuike/src/main/java/com/novelbook/android/utils/NovelParseUtil.java b/zhuike/src/main/java/com/novelbook/android/utils/NovelParseUtil.java index a66c592..40a5d43 100644 --- a/zhuike/src/main/java/com/novelbook/android/utils/NovelParseUtil.java +++ b/zhuike/src/main/java/com/novelbook/android/utils/NovelParseUtil.java @@ -1,19 +1,29 @@ package com.novelbook.android.utils; -import com.novelbook.android.db.Chapter; +import android.text.TextUtils; +import android.util.Log; +import com.novelbook.android.db.Chapter; +import com.novelbook.android.netutils.HttpMethods; + +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -public class NovelParseUtil { +import okhttp3.Request; +import okhttp3.Response; +public class NovelParseUtil { + private static final String TAG=NovelParseUtil.class.getSimpleName(); private static final String A_Regex =Constants.A_Regex ;//"]+href[\\s]*=[\\s]*['\"]?([^'\"]+)['\"\\s]?[^>]*>([^<]+)<"; public static String[] getChaptersArray(String muluUrl, String html, JSONObject siteJson) throws JSONException { @@ -59,13 +69,13 @@ public class NovelParseUtil { String chapterUrlRegexOnMulu = siteJson.getString("chapterUrlRegexOnMulu"); String chapterUrlPattern = siteJson.getString("chapterUrlPattern"); - + Log.d(TAG, "getChaptersMap: chapterUrlRegexOnMulu: " + chapterUrlRegexOnMulu); Map muluMap = new LinkedHashMap(); String regex = A_Regex; if (!isBlank(chapterUrlRegexOnMulu)) { regex = chapterUrlRegexOnMulu; } - + Log.d(TAG, "getChaptersMap: regex: " + regex); String[] rows = REUtil.matchs(regex, html);; if (rows == null || rows.length == 0) return null; for (int i = rows.length - 1; i >= 0; i--) { @@ -114,6 +124,132 @@ public class NovelParseUtil { return text.trim(); } + + + + public static String[] getChapters(String url, JSONObject siteJson) throws JSONException { + //if (!siteJson.keys().("chapterUrlRegexOnMulu")) return null; + String chapterUrlRegexOnMulu = siteJson.getString("chapterUrlRegexOnMulu"); + // if(TextUtils.isEmpty(chapterUrlRegexOnMulu)) return null; + if(!siteJson.has("chapterUrlRegexOnMulu")){ + return null ; + } + JSONArray muluArray = siteJson.getJSONArray("chapterUrlRegexOnMulu"); + if (muluArray == null || muluArray.length()== 0) return null; + + Map context = new HashMap(); + context.put("url", url); + List result = new ArrayList(); + // 最外部的大的规则对象 + for (int i = 0; i < muluArray.length(); i++) { + JSONObject regexsJson = muluArray.getJSONObject(i); + String source = regexsJson.getString("source"); + source = getContent(source, context); + if (source.startsWith("html:")) { + String _url = source.substring("html:".length()); + source = access(_url); + } + + // 第一次Regex对象 + JSONArray regexsArray = regexsJson.getJSONArray("regexs"); + for (int j = 0; j < regexsArray.length(); j++) { + JSONObject regexJson = regexsArray.getJSONObject(j); + String[] values = null; + if (regexJson.getBoolean("group")) { + values = REUtil.groups(regexJson.getString("regex"), source); + } else { + values = REUtil.matchs(regexJson.getString("regex"), source); + } + if (values != null) context.put(regexJson.getString("name"), values); + + // String child = siteJson.getString("child"); + + if ( siteJson.has("child") ) { + // 一般用来做返回结果用的 + JSONObject childJson = regexJson.getJSONObject("child"); + for (String value : values) { + String[] values2 = null; + if (childJson.getBoolean("group")) { + values2 = REUtil.groups(childJson.getString("regex"), value); + } else { + values2 = REUtil.matchs(childJson.getString("regex"), value); + } + if (values != null) { + context.put(childJson.getString("name"), values2); + if (siteJson.has("output")) { + JSONArray outputArray = childJson.getJSONArray("output"); + for (int m = 0; m < outputArray.length(); m++) { + String v = getContent(outputArray.getString(m), context); + if (v != null) result.add(v); + } + } + } + } + } + } + } + + return result.toArray(new String[0]); + } + + private static String getContent(String var, Map context) { + String[] vs = REUtil.matchs("\\{[^\\}]+?\\}", var); + for (String v : vs) { + String vn = v.substring(1, v.length() - 1); + String value = ""; + if (vn.endsWith("]")) { + int pos = vn.indexOf("["); + if (pos == -1) continue; + int index =Integer.parseInt(vn.substring(pos + 1, vn.length() - 1)); + Object ov = context.get(vn.substring(0, pos)); + if (ov == null) continue; + if (ov.getClass().isArray()) { + String[] ovs = (String[])ov; + if (index >= ovs.length) continue; + value = ovs[index]; + } + } else { + if (context.get(vn) == null) continue; + value = context.get(vn).toString(); + } + + int pos = var.indexOf(v); + var = var.substring(0, pos) + value + var.substring(pos + vn.length() + 2); + } + + return var; + } + + + private static String access(String url) { + Request.Builder builder = new Request.Builder() + // .tag(mNovel.getNovelId()) //标记 请求的tag,切换小说或离开小说界面(BookActivity) 时 取消未执行完毕的 此tag的所有请求 + .url(url) + .removeHeader("Pragma") + .header("Cache-Control", "public, max-age=" + 0) + // .header("Accept-Encoding","gzip, deflate, sdch") + .header("Accept-Language","zh-CN,zh;q=0.9") + .header( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") + .header( "Upgrade-Insecure-Requests", "1") + ; + + Request request =builder.build() ; + try { + Response response = HttpMethods.getOkClient().newCall(request).execute(); + String s =response.body().string(); + Log.d(TAG, "prepare book access: " +s); + return s; + } catch (IOException e) { + e.printStackTrace(); + } + return ""; + + } + + + + + private static boolean isBlank(String value) { return value == null || "".equals(value); }